From b1ea30925dff751eced23dfa94ff578a20ea0b94 Mon Sep 17 00:00:00 2001 From: "Field G. Van Zee" Date: Fri, 23 Feb 2018 17:42:48 -0600 Subject: [PATCH 01/53] CHANGELOG update (0.3.0) Change-Id: Id038b00a62de51c9818ad249651ec5dc662f4415 --- CHANGELOG | 3581 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 3577 insertions(+), 4 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index c9a04cbde..4b8218ffb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,10 +1,2863 @@ -commit 940a707ac78de975110e17c95765e65b89aa5e10 (HEAD -> master, tag: 0.2.2) +commit 709f8361ebc90b96b02ebe5c5ffb6fc3b1b25e58 (HEAD -> master, tag: 0.3.0) +Author: Field G. Van Zee +Date: Fri Feb 23 17:42:48 2018 -0600 + + Version file update (0.3.0) + +commit 3defc7265c12cf85e9de2d7a1f243c5e090a6f9d (origin/master, origin/HEAD) +Author: Field G. Van Zee +Date: Fri Feb 23 17:38:19 2018 -0600 + + Applied 34b72a3 to non-active/unused microkernels. + + Details: + - Applied the read-beyond-bounds bugfix in 34b72a3 to other haswell and + zen kernels (ie: other microtile shapes) which are not used by default. + This was done mostly in case someone decided to pick up these kernels + and start using them, not because it affects BLIS's behavior + out-of-the-box. + +commit 34b72a351745aa0d47bb0b74ebcd0f0a616d613d +Author: Field G. Van Zee +Date: Fri Feb 23 16:33:32 2018 -0600 + + Fixed obscure read-beyond-bounds bug in sgemm ukrs. + + Details: + - Fixed an obscure bug in the bli_sgemm_haswell_asm_6x16 and + bli_sgemm_zen_asm_6x16 microkernels when the input/output matrix C + is stored with general stride (ie: both rs and cs are non-unit). The + bug was rooted in the way those microkernels read from matrix C-- + namely, they used vmovlps/vmovhps instead of movss. By loading two + floats at a time, even if one of them was treated as junk, the + assembly code could be written in a more concise manner. However, + under certain conditions--if m % mr == 0 and n % nr == 0 and the + underlying matrix is not an internal "view" into a larger matrix-- + this could result in the very last vmovhps of the last (bottom-right) + microkernel invocation reading beyond valid memory. Specifically, the + low 32 bits read would always be valid, but the high 32 bits could + reside beyond the bounds of the array in which the output C matrix is + contained. To remedy this situation, we now selectively use movss to + load any element that could be the last element in the matrix. + +commit 5112e1859e7f8888f5555eb7bc02bd9fab9b4442 (origin/rt, rt) +Author: Field G. Van Zee +Date: Fri Feb 23 14:31:26 2018 -0600 + + Added missing 'restrict' to some kernels' cntx_t*. + + Details: + - Added missing 'restrict' keyword to cntx_t* argument of function + signatures corresponding to level-1v, level-1f, and level-1m kernels. + This affected bli_l1v_ker_prot.h, bli_l1f_ker_prot.h, and + bli_l1m_ker_prot.h. (The 'restrict' was already being used to + qualify cntx_t* arguments for kernels defined in bli_l3_ker_prot.h.) + - Added comments to bli_l1v_ker.h, bli_l1f_ker.h, bli_l1m_ker.h, and + bli_l3_ukr.h that help explain how those headers function to produce + kernel prototypes using the prototype macros defined in the files + mentioned above. + +commit 1fa8af95d807168e0849adb668492601e7009be0 +Merge: c084b03b 16813335 +Author: Field G. Van Zee +Date: Wed Feb 21 17:54:02 2018 -0600 + + Merge branch 'rt' + +commit c084b03b31d84427a120e391963db5419f1911ee +Merge: 5d03b6e6 fa74af4e +Author: Field G. Van Zee +Date: Wed Feb 21 17:52:17 2018 -0600 + + Merge branch 'rt' + +commit 16813335bdb5978bc9a26cd00a32bd5a130130c4 +Merge: fa74af4e 5a7005dd +Author: Field G. Van Zee +Date: Wed Feb 21 17:43:32 2018 -0600 + + Merge branch 'amd' into rt + + Details: + - Merged contributions made by AMD via 'amd' branch (see summary below). + Special thanks to AMD for their contributions to-date, especially with + regard to intrinsic- and assembly-based kernels. + - Added column storage output cases to microkernels in + bli_gemm_zen_asm_d6x8.c and bli_gemmtrsm_l_zen_asm_d6x8.c. Even with + the extra cost of transposing the microtile in registers, this is + much faster than using the general storage case when the underlying + matrix is column-stored. + - Added s and d assembly-based zen gemmtrsm_u microkernel (including + column storage optimization mentioned above). + - Updated zen sub-configuration to reflect presence of new native + kernels. + - Temporarily reverted zen sub-configuration's level-3 cache blocksizes + to smaller haswell values. + - Temporarily disabled small matrix handling for zen configuration + family in config/zen/bli_family_zen.h. + - Updated zen CFLAGS according to changes in 1e4365b. + - Updated haswell microkernels such that: + - only one vzeroupper instruction is called prior to returning + - movapd/movupd are used in leiu of movaps/movups for double-real + microkernels. (Note that single-real microkernels still use + movaps/movups.) + - Added kernel prototypes to kernels/zen/bli_kernels_zen.h, which is + now included via frame/include/bli_arch_config.h. + - Minor updates to bli_amaxv_ref.c (and to inlined "test" implementation + in testsuite/src/test_amaxv.c). + - Added early return for alpha == 0 in bli_dotxv_ref.c. + - Integrated changes from f07b176, including a fix for undefined + behavior when executing the 1m method under certain conditions. + - Updated config_registry; no longer need haswell kernels for zen + sub-configuration. + - Tweaked marginal and pass thresholds for dotxf. + - Reformatted level-1v, -1f, and -3 amd kernels and inserted additional + comments. + - Updated LICENSE file to explicitly mention that parts are copyright + UT-Austin and AMD. + - Added AMD copyright to header templates in build/templates. + + Summary of previous changes from 'amd' branch. + - Added s and d assembly-based zen gemm microkernels (d6x8 and d8x6) and + s and d assembly-based zen gemmtrsm_l microkernels (d6x8). + - Added s and d intrinsics-based zen kernels for amaxv, axpyv, dotv, dotxv, + and scalv, with extra-unrolling variants for axpyv and scalv. + - Added a small matrix handler to bli_gemm_front(), with the handler + implemented in kernels/zen/3/bli_gemm_small_matrix.c. + - Added additional logic to sumsqv that first attempts to compute the + sum of the squares via dotv(). If there is a floating-point exception + (FE_OVERFLOW), then the previous (numerically conservative) code is + used; otherwise, the result of dotv() is square-rooted and stored as + the result. This new implementation is only enabled when FE_OVERFLOW + is #defined. If the macro is not #defined, then the previous + implementation is used. + - Added axpyv and dotv standalone test drivers to test directory. + - Added zen support to old cpuid_x86.c driver in build/auto-detect/old. + - Added thread-local and __attribute__-related macros to bli_macro_defs.h. + +commit 5d03b6e6e19d5a07f0cccf1a158f02fbd62dfd99 +Author: Devin Matthews +Date: Mon Feb 19 11:31:30 2018 -0600 + + Fix asm macro include line for KNL. Fixes #167. + +commit f07b176c84dc9ca38fb0d68805c28b69287c938a +Author: Field G. Van Zee +Date: Thu Feb 15 18:36:54 2018 -0600 + + Fixed an obscure bug in the 1m implementation. + + Details: + - Fixed a bug in the way the bli_gemm1m_cntx_ref() function (defined in + ref_kernels/bli_cntx_ref.c) initializes its context for 1m execution. + Previously, the function probed the context that was in the process of + being updated for use with 1m--this context being previously + initialized/copied from a native context--for its storage preference + to determine which "variant" (row- or column-oriented) of 1m would be + needed. However, the _cntx_ref() function was not updating the method + field of the context until AFTER this query, and the conditional which + depended on it, had taken place, meaning the storage preference query + function would mistakenly think the context was for native execution, + since the context's method field would still be set to BLIS_NAT. This + would lead it to incorrectly grab the storage preference of the complex + domain microkernel rather than the corresponding real domain + microkernel, which could cause the storage preference predicate to + evaluate to the wrong value, which would lead to the _cntx_ref() + function choosing the wrong variant. This could lead to undefined + behavior at runtime. The method is now explicitly set within the + context prior to calling the storage preference query function. + - Updated comments in frame/ind/oapi/bli_l3_3m4m1m_oapi.c. + - Fixed a typo in the commented-out CFLAGS in config/zen/make_defs.mk, + which are appropriate for gcc 6.x and newer. (Mistakenly used + -march=bdver4 instead of -march=znver1.) + +commit 1f94bb7b96eb2b67257e6c4df89e29c73e9ab386 +Author: Field G. Van Zee +Date: Fri Jan 19 12:46:53 2018 -0600 + + Document how to enable zen-specific instructions. + + Details: + - Added as a comment in config/zen/make_defs.mk the list of compiler flags + that could be added to manually enable the instructions provided by the + Zen microarchitecture that are not already implied by -march=bdver4. + This information, along with the previous commit's flags to selectively + disable Bulldozer instructions no longer present in Zen, was gathered + from [1]. I hesitate to enable use of these instructions since I don't + have any Zen hardware to test on yet. + [1] https://wiki.gentoo.org/wiki/Ryzen + +commit 1e4365b21bafa02bd108c5ac4705a25671fb9441 +Author: Field G. Van Zee +Date: Thu Jan 18 12:03:51 2018 -0600 + + Augment zen CFLAGS to prevent illegal instruction. + + Details: + - Added various compiler flags (-mno-fma4 -mno-tbm -mno-xop -mno-lwp) so + that compiling with -march=bdver4 on zen-based architectures does not + result in an illegal instruction error at runtime. Note: This fix is + only needed for gcc 5.4; gcc 6.3 or later supports the use of + -march=znver1, which can be used in lieu of the augmented set of flags + based on bdver4. Thanks to Nisanth Padinharepatt for reporting this + error. + +commit fa74af4e1fa7385ac3f3089fe1ea7bb88c906029 +Author: Field G. Van Zee +Date: Tue Jan 9 13:43:15 2018 -0600 + + Minor labeling update for './configure -c' output. + + Details: + - Print the name of the configuration in the output of the + kernel-to-config map (and chosen pairs list) as a subtle way to remind + the user that these only apply to the targeted configuration (whereas + the config list and kernel list are printed without regard to which + configuration was actually targeted). + +commit 5cdea756c7391e2c6cbfb38436ef9a205f860237 +Merge: 9d8858b5 1e7a4896 +Author: Field G. Van Zee +Date: Sun Jan 7 19:45:20 2018 -0600 + + Merge branch 'rt' + +commit 9d8858b5cff4a4b078b87872847a5710073fff0a +Merge: 0b3ca3cf f7df64da +Author: Devin Matthews +Date: Sun Jan 7 10:03:25 2018 -0600 + + Merge pull request #164 from devinamatthews/master + + Don't use memkind for skx configuration. + +commit f7df64daf6bbe6431effada6e13d8d1fab5aa221 +Author: Devin Matthews +Date: Sun Jan 7 09:37:25 2018 -0600 + + Don't use memkind for skx configuration. Fixes #163. + +commit 1e7a4896e0cbe73c4685fa956278e3f28273cdf9 +Author: Field G. Van Zee +Date: Fri Jan 5 12:33:48 2018 -0600 + + Minor error handling in update-version-file.sh. + + Details: + - Added explicit handling of situations when 'git describe --tags' + returns an error. This command is used by update-version-file.sh + when deciding whether or not to update the version file prior to + configuration. + - Removed bli_packm.c and bli_unpackm.c, as they contained no source + code. + +commit 0b3ca3cfb682715a3686fd93ebb10d4a695d1162 +Author: Field G. Van Zee +Date: Thu Jan 4 20:51:35 2018 -0600 + + Intelligently select compiler for auto-detection. + + Details: + - Rewrote code that selects the compiler for the purposes of compiling + the auto-detection executable. CC (if specified) is tried first. Then + gcc. Then clang. The absolute fallback is cc. The previous code was + sort of broken, and seemed to unintentionally always use gcc. + - Moved various configuration-agnostic flags from config/*/make_defs.mk + files to common.mk. The new mechanism appends the configuration- + agnostic flags to the various compiler flag variables initialized in + make_defs.mk. Flags specific to the sub-configuration are still set + in make_defs.mk. + - Added -Wno-tautological-compare to CMISCFLAGS when clang is in use. + Also added the flag to the compiler instantiation during configure- + time hardware detection (when clang is selected). + - Added some missing (but mostly-optional) quotes to configure script. + +commit 5a7005dd44ed3174abbe360981e367fd41c99b4b (origin/amd, amd) +Merge: 7be88705 3bc99a96 +Author: Nisanth M P +Date: Wed Jan 3 12:05:12 2018 +0530 + + Merge changes in AMD beta release 0.95 into amd branch + +commit 0b9c5127e91508c115228ca604ee2dac8de8f477 +Author: Field G. Van Zee +Date: Sat Dec 23 15:53:44 2017 -0600 + + Enabled C99, added stdint.h to auto-detect build. + + Details: + - Added "-std=c99" to compiler arguments when building auto-detection + driver in configure script. + - Added #include to all three source files needed by auto- + detection program. + +commit 0ce5e19c318e04909d3e664d69accb3a0fc6b988 +Author: Field G. Van Zee +Date: Sat Dec 23 15:32:03 2017 -0600 + + Reimplemented configure-time hardware detection. + + Details: + - Reimplemented the hardware detection functionality invoked when running + "./configure auto". Previously, a standalone script in build/auto-detect + that used CPUID was used. However, the script attempted to enumerate all + models for each microarchitecture supported. The new approach recycles + the same code used for runtime hardware detection introduced in 2c51356. + This has two immediate benefits. First, it reduces and consolidates the + code required to detect microarchitectures via the CPUID instruction. + Second, it provides an indirect way of testing at configure-time the + code that is used to detect hardware at runtime. This code is (a) only + activated when targeting a configuration family (such as intel64 or + amd64) at configure-time and (b) somewhat difficult to test in + practice, since it relies on having access to older microarchitectures. + - The above change required placing conditional cpp macro blocks in + bli_arch.c and bli_cpuid.c which either #include "blis.h" or #include + a bare-bones set of headers that does not rely on the presence of a + bli_config.h header. This is needed because bli_config.h has not been + created yet when configure-time auto-detection takes places. + - Defined a new function in bli_arch.c, bli_arch_string(), which takes + an arch_t id and returns a pointer to a string that contains the + lowercase name of the corresponding microarchitecture. This function + is used by the auto-detection script to printf() the name of the + sub-configuration corresponding to the detected hardware. + +commit 9804adfd405056ec332bb8e13d68c7b52bd3a6c1 (origin/selfinit, selfinit) +Author: Field G. Van Zee +Date: Thu Dec 21 19:22:57 2017 -0600 + + Added option to disable pack buffer memory pools. + + Details: + - Added a new configure option, --[en|dis]able-packbuf-pools, which will + enable or disable the use of internal memory pools for managing buffers + used for packing. When disabled, the function specified by the cpp + macro BLIS_MALLOC_POOL is called whenever a packing buffer is needed + (and BLIS_FREE_POOL is called when the buffer is ready to be released, + usually at the end of a loop). When enabled, which was the status quo + prior to this commit, a memory pool data structure is created and + managed to provide threads with packing buffers. The memory pool + minimizes calls to bli_malloc_pool() (i.e., the wrapper that calls + BLIS_MALLOC_POOL), but does so through a somewhat more complex + mechanism that may incur additional overhead in some (but not all) + situations. The new option defaults to --enable-packbuf-pools. + - Removed the reinitialization of the memory pools from the level-3 + front-ends and replaced it with automatic reinitialization within the + pool API's implementation. This required an extra argument to + bli_pool_checkout_block() in the form of a requested size, but hides + the complexity entirely from BLIS. And since bli_pool_checkout_block() + is only ever called within a critical section, this change fixes a + potential race condition in which threads using contexts with different + cache blocksizes--most likely a heterogeneous environment--can check + out pool blocks that are too small for the submatrices it wishes to + pack. Thanks to Nisanth Padinharepatt for reporting this potential + issue. + - Removed several functions in light of the relocation of pool reinit, + including bli_membrk_reinit_pools(), bli_memsys_reinit(), + bli_pool_reinit_if(), and bli_check_requested_block_size_for_pool(). + - Updated the testsuite to print whether the memory pools are enabled or + disabled. + +commit 107801aaae180c00022f1b990bc59038c14949d2 +Merge: d9c05745 0084531d +Author: Field G. Van Zee +Date: Mon Dec 18 16:29:28 2017 -0600 + + Merge branch 'master' into selfinit + +commit 0084531d3eea730a319ecd7018428148c81bbba7 +Author: Field G. Van Zee +Date: Sun Dec 17 18:58:25 2017 -0600 + + Updated flatten-headers.py for python3. + + Details: + - Modifed flatten-headers.py to work with python 3.x. This mostly + amounted to removing print statements (which I replaced with calls + to my_print(), a wrapper to sys.stdout.write()). Thanks to Stefan + Husmann for pointing out the script's incompatibility with python 3. + - Other minor changes/cleanups. + +commit 90b11b79c302f208791bdfb1ed754873103c7ce5 +Author: Field G. Van Zee +Date: Sun Dec 17 17:34:32 2017 -0600 + + Modest performance boost to flatten-headers.py. + + Details: + - Updated flatten-headers.py to pre-compile the main regular expression + used to isolate #include directives and the header filenames they + reference. The compiled regex object is then used over and over on + each header file in the tree of referenced headers. This appears to + have provided a 1.7-2x performance increase in the best case. + - Other minor tweaks, such as renaming the main recursive function from + replace_pass() to flatten_header(). + +commit 99dee87f30b4d437fa6b5e4ba862526d07b9f08b +Author: Field G. Van Zee +Date: Sun Dec 17 16:47:27 2017 -0600 + + Reimplemented flatten-headers.sh in python. + + Details: + - Added flatten-headers.py, a python implementation of the bash script + flatten-headers.sh. The new script appears to be 25-100x faster, + depending on the operating system, filesystem, etc. The python script + abides by the same command line interface as its predecessor and + targets python 2.7 or later. (Thanks to Devin Matthews for suggesting + that I look into a python replacement for higher performance.) + - Activated use of flatten-headers.py in common.mk via the FLATTEN_H + variable. + - Made minor tweaks to flatten-headers.sh such as spelling corrections + in comments. + +commit d9c0574599c3f97c0f9b6c334a077bab9452e1f4 +Author: Field G. Van Zee +Date: Thu Dec 14 17:13:42 2017 -0600 + + Allow travis failures of OS X builds that run testsuite. + + Details: + - Added an allowance for OS X builds that run the testsuite to fail. + There seems to be an issue with 1m when running in Travis CI under + OS X and clang, but only in double-precision. Haven't been able to + reproduce the error on my own, and thus, I can't debug it. (Hopefully + it is simply a version-specific compiler bug.) + +commit 86cd23b7379b00a42b4ecc04fa668f1e3f9b54ee +Author: Field G. Van Zee +Date: Thu Dec 14 15:47:41 2017 -0600 + + Fixed testsuite Makefile brokenness from 9091a207. + + Details: + - Fixed a makefile error encountered when building the testsuite directly + in its directory (as opposed to indirectly via 'make test'). The fix + involves introducing a new variable, BUILD_PATH, alongside the existing + DIST_PATH variable. By default, BUILD_PATH is set to the current + directory, and is overridden by other Makefiles used by, for example, + the testsuite and standalone test drivers in testsuite or test, + respectively. + - Some files/directories in common.mk were redefined in terms of + BUILD_DIR, such as the locations of config.mk file and the intermediate + include directory. + +commit 6a3a8924c04d25507fc4aa593df30c56c7dc12f7 +Author: Field G. Van Zee +Date: Thu Dec 14 13:20:02 2017 -0600 + + Temporarily show Makefile's testsuite output. + + Details: + - Disabled redirection of testsuite output for 'test' target. This is + part of an attempt to debug a segmentation fault on OS X via Travis. + +commit 9a01080dd426915bed18229f70401bfa639dc283 +Merge: 83316485 a32e8a47 +Author: Field G. Van Zee +Date: Thu Dec 14 11:27:19 2017 -0600 + + Merge branch 'master' into selfinit + +commit a32e8a47c022b6071302b2956af5728976c83ca9 (origin/travis) +Author: Field G. Van Zee +Date: Wed Dec 13 16:31:36 2017 -0600 + + Added an exclusion to .travis.yml. + + Details: + - Added exclusion for out-of-tree builds on OS X (clang). + +commit b9f7d987df548965c86e16e0ba94d5cad0d9b399 +Author: Field G. Van Zee +Date: Wed Dec 13 16:22:09 2017 -0600 + + Cleaned up after previous travis oot debugging. + + Details: + - Removed debugging output from common.mk related to Travis CI + out-of-tree builds. + - Other minor cleanups to common.mk. + +commit 9091a207aa8c49e279676ea02be533480b3b0d5a +Author: Field G. Van Zee +Date: Wed Dec 13 16:12:34 2017 -0600 + + Attempted fix to travis oot build failure. + + Details: + - Found the likely cause of the Travis CI out-of-tree build failures: + config.mk was being read from DIST_PATH, rather than the current + directory. + +commit c01c71c33e236e6c91f5ddd3ec1e3faec89368c1 +Author: Field G. Van Zee +Date: Wed Dec 13 15:58:50 2017 -0600 + + Added debugging output to Makefile. + + Details: + - Added $(info ...) statements in key locations in an attempt to reveal + why Travis CI doesn't like building BLIS out-of-tree. + +commit 784289d69dd6b3692444d3b3e290f6a014465b72 +Author: Field G. Van Zee +Date: Wed Dec 13 15:31:27 2017 -0600 + + Updated SHELL in common.mk from /bin/bash to bash. + +commit d9bb1d1d4ebc89ea75d9d927d09882162a914f77 +Author: Field G. Van Zee +Date: Wed Dec 13 15:27:54 2017 -0600 + + Defined SHELL in common.mk so "echo -n" works. + + Details: + - Defined the SHELL variable in common.mk as "/bin/bash" so that the + -n option can be used with echo in the Makefile rule for flattening + blis.h. Thanks to Devin Matthews for suggesting this fix. + +commit 9289a08667df2044f3a37af54d893efe2b56d555 +Author: Field G. Van Zee +Date: Wed Dec 13 15:14:27 2017 -0600 + + Attempt 3 on .travis.yml. + +commit 720bfcf0ef54fdc41df0dcaa94503edb0d5c8972 +Author: Field G. Van Zee +Date: Wed Dec 13 14:52:28 2017 -0600 + + More fixes to .travis.yml. + + Details: + - Fixed a mistake (hopefully) in d0c4dd0 that resulted in many more + osx/clang sub-tests than intended. + - Shortened the variable names in an effort to make them more readable + via the Travis CI web interface. + +commit 8717c9c97fe9b1ecd3b3192049a73976f8390ca7 +Author: Field G. Van Zee +Date: Wed Dec 13 14:36:37 2017 -0600 + + Added 'pwd' commands to .travis.yml for debugging. + + Details: + - Added 'pwd' commands to the script portion of the .travis.yml file in + an attempt to uncover the problem with the recent out-of-tree build + testing changes made in d0c4dd0. + +commit 83316485ce10f6fcafe92a1c146282de0dd8068a +Author: Field G. Van Zee +Date: Wed Dec 13 14:14:50 2017 -0600 + + Simplified/fixed self-initialization. + + Details: + - Fixed a race condition in self-initialization whereby the bli_is_init + static variable could be erroneously read as TRUE by thread 1 while + thread 0 is still executing bli_init_apis(), thus allowing thread 1 to + use the library before it is actually ready. Thanks to to Minh Quan Ho + and Devin Matthews for pointing out this issue. + - Part of the solution to the aforementioned race condition was involved + replacing the runtime initialization of the global scalar constants + (e.g., BLIS_ONE, BLIS_ZERO, etc.) in bli_const.c with a static + initialization of those same constants. This eliminates the need for + bli_const_init() altogether. (The static initialization is made concise + via preprocess macros.) + - Defined bli_gks_query_cntx_noinit(), which behaves just like + bli_gks_query_cntx(), except that it does not call bli_init_once(). This + function is called in lieu of bli_gks_query_cntx() in bli_ind_init() and + bli_memsys_init() so as to not result in any recursion into + bli_init_once(). + - Removed BLIS_ONE_HALF, BLIS_MINUS_ONE_HALF global scalar constants. + They have no use in BLIS or its test products, and we have little reason + to believe they are used by others. + - Removed testsuite/out file, which was accidentally committed as part + of 70640a3. + +commit 6526d1d4ae6dbfa854ca8d1e5f224cd6ab3fa958 +Author: Field G. Van Zee +Date: Tue Dec 12 13:50:43 2017 -0600 + + Added temp_dir argument to flatten-headers.sh. + + Details: + - Added "temp_dir" argument to flatten-headers.sh so that the caller can + specify where intermediate files should be created as the script runs. + - Updated flatten-headers.sh to create intermediate files in temp_dir + instead of alongside the corresponding source files. This should now + (once again) allow out-of-tree builds where the BLIS distribution is + read-only, or where the out-of-tree build is running concurrently with + another out-of-tree build. (Thanks to Devin Matthews for pointing out + the possibility of simultaneous out-of-tree builds.) + +commit 94755017c967630daf2e31c1f63ed5e88ab0d6ab +Merge: d0c4dd00 5cf7b0c4 +Author: Field G. Van Zee +Date: Tue Dec 12 12:50:41 2017 -0600 + + Merge branch 'master' of github.com:flame/blis + +commit d0c4dd000ff38acc249e8acf7e0655a523991695 +Author: Field G. Van Zee +Date: Tue Dec 12 12:47:53 2017 -0600 + + Added out-of-tree build test to .travis.yml file. + + Details: + - Modified .travis.yml file to include an out-of-tree build test (using + the "auto" configure target). Thanks to Devin Matthews for this + suggestion. + +commit 5cf7b0c4e52922069183a87dc2aa177419644e04 +Author: Devin Matthews +Date: Tue Dec 12 12:38:48 2017 -0600 + + Ignore blis.h.interm [ci skip] + +commit 8d8ff74d15b4a584929cec36034ba6d3c53f7d27 +Author: Field G. Van Zee +Date: Tue Dec 12 12:32:50 2017 -0600 + + Further attempt to fix out-of-tree builds. + + Details: + - Fix applied in 87978f6 was necessary but not sufficient to fix + out-of-tree builds. It turns out that using a source tree that had + already built the target erroneously gave the impression that + out-of-tree builds were working again, when in fact they were still + broken. The additional changes in this commit should complete the + fix that was started in the aforementioned commit. Thanks to Devin + Matthews and Shaden Smith for their help in isolating this issue. + +commit 70640a37109290b57c344083c00624e13c496e30 +Author: Field G. Van Zee +Date: Mon Dec 11 17:18:43 2017 -0600 + + Implemented library self-initialization. + + Details: + - Defined two new functions in bli_init.c: bli_init_once() and + bli_finalize_once(). Each is implemented with pthread_once(), which + guarantees that, among the threads that pass in the same pthread_once_t + data structure, exactly one thread will execute a user-defined function. + (Thus, there is now a runtime dependency against libpthread even when + multithreading is not enabled at configure-time.) + - Added calls to bli_init_once() to top-level user APIs for all + computational operations as well as many other functions in BLIS to + all but guarantee that BLIS will self-initialize through the normal + use of its functions. + - Rewrote and simplified bli_init() and bli_finalize() and related + functions. + - Added -lpthread to LDFLAGS in common.mk. + - Modified the bli_init_auto()/_finalize_auto() functions used by the + BLAS compatibility layer to take and return no arguments. (The + previous API that tracked whether BLIS was initialized, and then + only finalized if it was initialized in the same function, was too + cute by half and borderline useless because by default BLIS stays + initialized when auto-initialized via the compatibility layer.) + - Removed static variables that track initialization of the sub-APIs in + bli_const.c, bli_error.c, bli_init.c, bli_memsys.c, bli_thread, and + bli_ind.c. We don't need to track initialization at the sub-API level, + especially now that BLIS can self-initialize. + - Added a critical section around the changing of the error checking + level in bli_error.c. + - Deprecated bli_ind_oper_has_avail() as well as all functions + bli__ind_get_avail(), where is a level-3 operation + name. These functions had no use cases within BLIS and likely none + outside of BLIS. + - Commented out calls to bli_init() and bli_finalize() in testsuite's + main() function, and likewise for standalone test drivers in 'test' + directory, so that self-initialization is exercised by default. + +commit 70a64432ee5a7adbee10fb7ff6d7b608c1940a7a +Author: Field G. Van Zee +Date: Mon Dec 11 13:14:20 2017 -0600 + + Fixed off-by-one indexing in bli_cpuid.c. + + Details: + - In bli_cpuid.c, fixed an off-by-one indexing statement in vpu_count() + whereby a string-terminating NULL character, '\0', is written beyond + the bounds of the model_num string. + - Minor whitespace and formatting edits to bli_cpuid.c. + +commit 87978f6261a080d261d01f9acf4e9cc18855c833 +Author: Field G. Van Zee +Date: Mon Dec 11 12:49:03 2017 -0600 + + Fixed broken out-of-tree builds since 52f9e6f. + + Details: + - Added missing $(DIST_PATH)/ prefix to relative path to flatten-headers.sh + script in common.mk so that the script could be found during out-of-tree + builds. Thanks to Devin Matthews for reporting this bug. + +commit 513ef4d040f89a18dda5154e8c4cf1aaf7463999 +Author: Field G. Van Zee +Date: Mon Dec 11 12:35:59 2017 -0600 + + Various typecasting fixes, mis-typed enums, etc. + + Details: + - Fixed implicit typecasting of conj_t to trans_t in bli_[un]packm_cxk.c. + - Properly typecast integer arguments to match format specifier in various + calls to printf() in bli_l3_thrinfo.c, bli_cntx.c, bli_pool.c, and + bli_util_oapi.c. + - Fixed "unsigned less-than-comparison with zero" checks in bli_check.c, + bli_cntx.h. + - Fixed mis-typed enums in bli_cntx.c (e.g., l1mkr_t that should have been + l1fkr_t or l1vkr_t). + - Fixed instances of opid_t value BLIS_GEMM that should have been l3ukr_t + value BLIS_GEMM_UKR in bli_cntx_ref.c. + - NOTE: These issues were identified via compiler warnings when building + BLIS with clang on a rather old installation of OS X: + $ clang --version + Apple LLVM version 5.0 (clang-500.2.79) (based on LLVM 3.3svn) + Target: x86_64-apple-darwin15.2.0 + Thread model: posix + +commit 3bc99a96a3648f51b9acdc8a8c7e1cf4eb815459 +Merge: 3a441183 78199c53 +Author: prangana +Date: Mon Dec 11 12:53:03 2017 +0530 + + Fix merge conflicts after rebase with release branch + + Change-Id: I581b26c6d515f717ff0dce91c7c0c92553aa2630 + +commit 3a44118398955d6f872e01f73ae5bb4a4f8500f7 +Author: Nisanth M P +Date: Wed Nov 15 11:11:17 2017 +0530 + + Added AMD copyright line to the changed files in last 3 commits + + Change-Id: I37d5dbbbe1b199e07529610a5e9cc9e49d067c66 + +commit 268a56c06e94d1c388766dbfe81d54efbe432809 +Author: Field G. Van Zee +Date: Wed Nov 1 11:51:41 2017 -0500 + + Revert to default SIMD alignment for bulldozer. + + Details: + - Removed the default-overriding #define of BLIS_SIMD_ALIGN_SIZE set in + config/bulldozer/bli_kernel.h. Not sure where this value came from, but + it would seem to allow for insufficient starting address alignment for + any matrices created via bli_malloc_user(), such as via + bli_obj_create(). Thanks to Rene Sitt for reporting the behavior that + led us to this bug. + - This commit is a manual patch of the same fix made to the 'rt' branch + in 8f150f2. + +commit 510a6863e28277f9446abfb77f1aea9f01d37e7a +Author: Devin Matthews +Date: Mon Oct 30 10:04:42 2017 -0500 + + Fix CVECFLAGS for bulldozer config. + +commit c669716790bdda5d2b11ea0a026cbc121b228842 +Author: Nisanth M P +Date: Tue Oct 24 16:36:36 2017 +0530 + + Adding __attribute__((constructor/destructor)) for CLANG case. + + CLANG supports __attribute__, but its documentation doesn't + mention support for constructor/destructor. Compiling with + clang and testing shows that it does support this. + + Change-Id: Ie115b20634c26bda475cc09c20960d687fb7050b + +commit 24e64a9d0877d788357fc63d4b947e977f8697f7 +Author: Field G. Van Zee +Date: Wed Oct 18 13:41:25 2017 -0500 + + Removed a duplicate bli_avx512_macros.h header. + + Details: + - Removed a duplicate header file that was causing problems during + installation for the 'knl' configuration. Thanks to Victor Eijkhout + for reporting this issue. + +commit 9c0a3c4c0260cbfefb9f11532f46508b4fd19ec2 +Author: Nisanth M P +Date: Mon Oct 16 22:06:57 2017 +0530 + + Thread Safety: Move bli_init() before and bli_finalize() after main() + + BLIS provides APIs to initialize and finalize its global context. + One application thread can finalize BLIS, while other threads + in the application are stil using BLIS. + + This issue can be solved by removing bli_finalize() from API. + One way to do this is by getting bli_finalize() to execute by default + after application exits from main(). + + GCC supports this behaviour with the help of __attribute__((destructor)) + added to the function that need to be executed after main exits. + + Similarly bli_init() can be made to run before application enters main() + so that application need not call it. + + Change-Id: I7ce6cfa28b384e92c0bdf772f3baea373fd9feac + +commit 83f31253eb21c5ecd8a5907835e57720daae0b8b +Author: Nisanth M P +Date: Mon Oct 16 21:07:50 2017 +0530 + + Thread safety: Make the global induced method status array local to thread + + BLIS retains a global status array for induced methods, and provides + APIs to modify this state during runtime. So, one application thread + can modify the state, before another starts the corresponding + BLIS operation. + + This patch solves this issue by making the induced method status array + local to threads. + + Change-Id: Iff59b6f473771344054c010b4eda51b7aa4317fe + +commit e923402e68029be379a4297de3ac6fb155ffd928 +Author: sthangar +Date: Thu Sep 28 12:15:36 2017 +0530 + + The inner loop paralleization is turned off by default, the JR and IR loop parameters are set to 1 by default + + Change-Id: I8c3c2ecbbd636259f6ffb92768ec04148205c3e5 + +commit a64c15de19327c7595376d699be676c7003e850e +Author: Field G. Van Zee +Date: Tue Sep 26 19:02:53 2017 -0500 + + Fixed a pthread typo in previous commit. + + Details: + - Misnamed 'pthread_mutex_t' type in bli_memsys.c as 'thread_mutex_t'. + +commit 42dcd589c37e1a2473ab2e1539207da97aebc07f +Author: Field G. Van Zee +Date: Tue Sep 26 17:00:04 2017 -0500 + + Fixed bugs in gemm/gemmtrsm ukr tests in testsuite. + + Details: + - Fixed a bug in gemmtrsm test module that was due to improper partitioning + into a k x k triangular matrix for the purposes of obtaining an mr x k + micropanel of A with which to test. + - Fixed a bug in gemm and gemmtrsm test modules that would only manifest for + very large k (depending on the product of mr x kc on that architecture). + The bug arose from the fact that the test module was triggering the + allocation of blocks from the internal memory pools, which are limited in + size. This allocation imposes an implicit assumption that the micro- + panel being tested with will fit inside, and this assumption is violated + for large values of k. Arbitrarily large k may now be tested for both + operation tests. + - Added OpenMP/pthread critical sections around the setting or getting of + statuses from the induced method operation lookup table in bli_l3_ind.c. + - Added the 'static' keyword to all pthread_mutex_t global variables in BLIS. + - Thanks to Nisanth Padinharepatt of AMD for reporting the first and third + issues. + +commit 206beb68ff73b75f5c382413967aacbb8a0aac3a +Author: Field G. Van Zee +Date: Sat Sep 9 14:10:15 2017 -0500 + + Updated bibtex info for BLIS5 (3m4m) article. + +commit 0c8c0363aeb1f4aa88f7ec2d02403dab05a6e014 +Author: sthangar +Date: Mon Aug 28 16:44:42 2017 +0530 + + Bug fix for the testsuite build failing + + Change-Id: I7cd8c9d187387c48b2564e45cbfb8df985e93d77 + +commit 63d1c84465b50f64787808dd3e8494e683c16821 +Author: sthangar +Date: Wed Aug 23 13:01:14 2017 +0530 + + Adding auto hardware detection for Zen + + Change-Id: I40ce6705dd66b35000c4ccddffad1c5b65998caf + +commit 537fb2a895b09be94b11947696fd2da629be24dd +Author: Devin Matthews +Date: Tue Aug 15 10:02:25 2017 -0500 + + Add vzeroupper to Intel AVX kernels. + +commit 7628de3f76f78a44788807605a4601ddda445854 +Author: Field G. Van Zee +Date: Thu Aug 10 16:24:28 2017 -0500 + + Removed trailing enum commas from bli_type_defs.h. + + Details: + - Removed trailing commas from enums in bli_type_defs.h. Thanks to + Erling Andersen for pointing out this inconsistency and suggesting + the change. + +commit a666fd4e267ffae3d4b21f38d569c61ff56adc9e +Author: Field G. Van Zee +Date: Sat Aug 5 13:04:31 2017 -0500 + + Added edge handling to _determine_blocksize_b(). + + Details: + - Added explicit handling of situations where i == dim to + bli_determine_blocksize_b_sub(). This isn't actually needed by any + current use case within BLIS, but handling the situation is nonetheless + prudent. Thanks to Minh Quan for reporting this issue and requesting + the fix. + +commit 0c8afa546d7f33760415519ba328d7c49eb7aa06 +Author: Field G. Van Zee +Date: Fri Aug 4 14:17:44 2017 -0500 + + Fixed a minor bug in level-3 packm management. + + Details: + - Fixed a bug in bli_l3_packm() that caused cntl_t-cached packed mem_t + entries to be released and then re-acquired unnecessarily. (In essence, + the "<" operands in the conditional that guards the + release-and-reacquire code block simply needed to be swapped.) The bug + should have only affected performance (rather than the computed result). + Thanks to Minh Quan for identifying and reporting the bug. + +commit 6cf68a185d83fa46d438fcef65258ace78e24b13 +Author: Devin Matthews +Date: Mon Jul 31 15:19:51 2017 -0500 + + Change lsame_ signature to match lapacke. + +commit 6a9bd97295cc4fb1cbcd28f69824a43c073c9a76 +Author: Field G. Van Zee +Date: Sat Jul 29 20:17:05 2017 -0500 + + Fixed pthreads compile bug with previous commit. + + Details: + - Erroneously passed family parameter into l3int_t function despite + that function not taking the parameter. Oops. + +commit 95adc43d800431dc0a02ca83a51426dbef641ad6 +Author: Field G. Van Zee +Date: Sat Jul 29 14:53:39 2017 -0500 + + Moved 'family' field from cntx_t to cntl_t. + + Details: + - Removed the family field inside the cntx_t struct and re-added it to the + cntl_t struct. Updated all accessor functions/macros accordingly, as well + as all consumers and intermediaries of the family parameter (such as + bli_l3_thread_decorator(), bli_l3_direct(), and bli_l3_prune_*()). This + change was motivated by the desire to keep the context limited, as much + as possible, to information about the computing environment. (The family + field, by contrast, is a descriptor about the operation being executed.) + - Added additional functions to bli_blksz_*() API. + - Added additional functions to bli_cntx_*() API. + - Minor updates to bli_func.c, bli_mbool.c. + - Removed 'obj' from bli_blksz_*() API names. + - Removed 'obj' from bli_cntx_*() API names. + - Removed 'obj' from bli_cntl_*(), bli_*_cntl_*() API names. Renamed routines + that operate only on a single struct to contain the "_node" suffix to + differentiate with those routines that operate on the entire tree. + - Added enums for packm and unpackm kernels to bli_type_defs.h. + - Removed BLIS_1F and BLIS_VF from bszid_t definition in bli_type_defs.h. + They weren't being used and probably never will be. + +commit a98e4aa547f61ab09dd91d11478c2a2ef9882e11 +Author: Devin Matthews +Date: Thu Jul 20 14:50:13 2017 -0500 + + Clang can't make up it's mind what to support. + +commit 32eb36c3e8c2add2528514272044de16faed0c8f +Author: Devin Matthews +Date: Thu Jul 20 12:54:58 2017 -0500 + + Add default #define for __has_extension. + +commit 2a9aa134f7c29d3d4fdc160022ff257e61885a95 +Author: Devin Matthews +Date: Thu Jul 20 10:04:34 2017 -0500 + + Add fallbacks to __sync_* or __c11_atomic_* builtins when __atomic_* is not supported. Fixes #143. + +commit 6f07a034d575e1e9e30bb6417b8fcb77cf301297 +Author: Field G. Van Zee +Date: Wed Jul 19 15:40:48 2017 -0500 + + Updated ar option list used by all configurations. + + Details: + - Dropped 'u' from the list of modifiers passed into the library archiver + ar. Previously, "cru" was used, while now we employ only "cr". This + change was prompted by a warning observed on Ubuntu 16.04: + + ar: `u' modifier ignored since `D' is the default (see `U') + + This caused me to realize that the default mode causes timestamps to be + zero, and thus the 'u' option, which causes only changed object files to + be inserted, is not applicable. + +commit 32bc03f9eed8795cfd2f2615d1c9f8673e039c57 +Author: Field G. Van Zee +Date: Wed Jul 19 13:51:53 2017 -0500 + + Added --force-version=STRING option to configure. + + Details: + - Added an option to configure that allows the user to force an arbitrary + version string at configure-time. The help text also now describes the + usage information. + - Changed the way the version string is communicated to the Makefile. + Previously, it was read into the VERSION variable from the 'version' file + via $(shell cat ...). Now, the VERSION variable is instead set in + config.mk (via a configure-substituted anchor from config.mk.in). + +commit befaee6dd8b2a72de9e0461fe2ec1f36e9f88f3c +Author: Field G. Van Zee +Date: Tue Jul 18 17:56:00 2017 -0500 + + Updated openmp/pthread barriers with GNU atomics. + + Details: + - Updated the non-tree openmp and pthreads barriers defined in + bli_thrcomm_openmp.c and bli_thrcomm_pthreads.c to instead call a common + implementation in bli_thrcomm.c, bli_thrcomm_barrier_atomic(). This new + implementation goes through the same motions as the previous codes, but + protects its loads and increments with GNU atomic built-ins. These atomic + statements take memory ordering parameters that allow us to specify just + enough constraints for the barrier to work as intended on weakly-ordered + hardware. The prior implementation was only guaranteed to work on systems + with strongly- ordered memory. (Thanks to Devin Matthews for suggesting + this change and his crash-course in atomics and memory ordering.) + - Removed 'volatile' from structs' barrier field declarations in + bli_thrcomm_*.h. + - Updated bli_thrcomm_pthread.? files to use renamed struct barrier fields + consistent with that of the _openmp.? files. + - Updated other bli_thrcomm_* files to rename "communicator" variables to + simply "comm". + +commit 8f739cc847fcff2ddeeb336f8b2b9d080eb16f6c +Author: Field G. Van Zee +Date: Mon Jul 17 19:03:22 2017 -0500 + + Added API to set mt environment variables. + + Details: + - Renamed bli_env_get_nway() -> bli_thread_get_env(). + - Added bli_thread_set_env() to allow setting environment variables + pertaining to multithreading, such as BLIS_JC_NT or BLIS_NUM_THREADS. + - Added the following convenience wrapper routines: + bli_thread_get_jc_nt() + bli_thread_get_ic_nt() + bli_thread_get_jr_nt() + bli_thread_get_ir_nt() + bli_thread_get_num_threads() + bli_thread_set_jc_nt() + bli_thread_set_ic_nt() + bli_thread_set_jr_nt() + bli_thread_set_ir_nt() + bli_thread_set_num_threads() + - Added #include "errno.h" to bli_system.h. + - This commit addresses issue #140. + - Thanks to Chris Goodyer for inspiring these updates. + +commit 10163833075fd42be5b5b503acc855f91a484cfd +Author: Marat Dukhan +Date: Thu Jul 13 21:39:24 2017 -0700 + + Fix Emscripten builds + +commit c09b30d115eade72f44f37bf90aa848c9c0e79af +Author: Minh Quan HO +Date: Fri Jul 7 10:52:05 2017 +0200 + + set missing free_fp in bli_membrk_init for free-ing GEN_USE buffers + + The membrk's free_fp is called when releasing GEN_USE buffers, but this free_fp is + not set in bli_membrk_init + +commit 997628ed9793c72e9ef576dd8d715cfec27c4862 +Author: sthangar +Date: Fri Jun 30 12:23:19 2017 +0530 + + Reducing the framework overhead of GEMV routines + + Change-Id: I83607ad767bff74e305e915b54b0ea34ec3e5684 + +commit ee869066168239b710ad9938bb0e1ae454883f3a +Author: Kiran Varaganti +Date: Tue Jul 4 12:57:32 2017 +0530 + + Improved efficiency of dGEMM for large matrices by reducing TLB load misses and majorly L3 cache misses. This is achieved by changing the packed block sizes of matrix A & B. Now the optimum values are MC_D = 510 and KC_D = 1024. + + Change-Id: I2d8bdd5f62f2d1f8782ae2997f3d7a26587d1ca4 + +commit 7b933b90b1859c96de49a402d48de82909bc73e5 +Author: Devin Matthews +Date: Tue Jun 6 20:23:17 2017 -0500 + + Add new SSI acknowledgment + +commit 3485abba4b426fbf42b146a9611a0841f6d236c6 +Author: sthangar +Date: Wed May 24 11:48:16 2017 +0530 + + Checked in the small matrix code to compute GEMM called with A transpose case + + Change-Id: I29f40046d43d7a4b037c1cb322503ee26495f462 + +commit de16beb83b29b4b9748f70db985b0fe04db85f7d +Author: Devin Matthews +Date: Fri May 26 14:49:31 2017 -0400 + + PACKDIM_MR=8 didn't work out, but messing with the prefetching helps 2%. + +commit 25d0e618544b6eea7d3f13c7aec513ac0139801d +Author: Devin Matthews +Date: Fri May 26 14:47:36 2017 -0400 + + Revert "Change PACKDIM_MR (double) for haswell to 8." + + This reverts commit 681eec913d7c2ebcff637cec5c1627ced9a92b99. + +commit c5bdd84b35bc2a8ebf55b7763fb56c0c945be0cb +Author: Devin Matthews +Date: Fri May 26 12:28:09 2017 -0500 + + Change PACKDIM_MR (double) for haswell to 8. + +commit 172789d562001293b973bbdd8015bd27d37292e8 +Author: Field G. Van Zee +Date: Wed May 17 13:03:52 2017 -0500 + + Restored deleted lines from makefile fragments. + +commit 3ea9bd2c8e90dbd35655fa6a5b953dfea1f308fe +Author: Devin Matthews +Date: Wed May 17 12:29:44 2017 -0500 + + Change to /bin/sh. + + All scripts checked with Debian's checkbashisms. Also check for clang first in auto-detect.sh. + +commit 49438409eedb98d3f0ebf00b8d1eee0ae45f4f8c +Author: Devin Matthews +Date: Wed May 17 12:27:14 2017 -0500 + + Remove shebangs from makefiles. + +commit 497e2640474c016d576dce3530fa6a66891642a0 +Author: J M Dieterich +Date: Tue May 16 23:11:22 2017 -0400 + + Fix if/else structure. Thanks to TravisCI. + +commit 835035c56a8de36ad25bb8d1375db170d489ef57 +Author: J M Dieterich +Date: Tue May 16 22:23:27 2017 -0400 + + Mark piledriver compilable w/ clang. + +commit 6cdb533472ee61af297c1f948307abbf45828887 +Author: J M Dieterich +Date: Tue May 16 22:12:12 2017 -0400 + + Mark bulldozer compilable w/ clang. + +commit a85697d62272da06d28cd1c947f6cf1098df6467 +Author: J M Dieterich +Date: Tue May 16 22:06:59 2017 -0400 + + Correct error message. + +commit e0c64cad271058688a2b999caf8c2767dc3aef7e +Author: J M Dieterich +Date: Tue May 16 22:03:23 2017 -0400 + + Indeed once can compile for carrizo also using clang. + +commit 4aafe0505d3f0954d095ded5459a76976e5093b4 +Author: J M Dieterich +Date: Tue May 16 21:50:49 2017 -0400 + + A bunch of shebang fixes from unportable /bin/bash to portable /usr/bin/env bash + +commit abaeaa68ea11e84be1810f564d6f38d506cbeb6a +Author: Field G. Van Zee +Date: Fri May 5 15:06:56 2017 -0500 + + Fixed a bug in norm1v, norm1m. + + Details: + - Fixed a bug that manifested as improperly-computed 1-norm for vectors + and matrices. This is one of the few operations in BLIS that does not + have its own test module within the testsuite, hence why it went + undetected for so long. The bad 1-norms were being used to normalize + matrices in the testsuite after initialization, which led to some + matrices containing a combination of "large" and "small" values. This + tended to push the residuals computed after each test away from zero. + In some cases, they were off *just* enough to the testsuite to label + it a "failure". Many thanks to Jeff Hammond for reporting this bug. + (Wonky details: the bug was due to improperly-defined level-0 scalar + macros for abval2, an operation that computes the absolute square, + or complex magnitude/modulus. Certain complex domain instances of + abval2 were being incorrectly defined in terms of real-only solutions, + leading to bad results. This level-0 operation forms the basis of + norm1v/norm1m. absq2 was also affected, but almost nothing uses + this operation.) + +commit cc3107ae1c2074f72b724aa748d2e5b4cb290ed5 +Author: Devin Matthews +Date: Thu May 4 10:35:22 2017 -0500 + + Setting any one of BLIS_NT_[IJ][CR] overrides BLIS_NUM_THEADS. Missing BLIS_NT_XX's are defaulted to 1. Fixes #123. + +commit c8ab91f70d399ee14edd30a3a5c46b24c5d2f910 +Author: Field G. Van Zee +Date: Wed May 3 15:04:51 2017 -0500 + + Disable complex 3m/4m in testsuite by default. + + Details: + - Disabled testsuite tests of all level-3 implementations based on 3m + and 4m. This will improve testing runtime on Travis CI as well as for + anyone manually running the testsuite using default test parameters. + Thanks to Devin Matthews for suggesting this change. + +commit 9700f0e5785007ddafb72a5ca83800dee61fd35c +Author: Jeff Hammond +Date: Tue May 2 19:25:21 2017 -0700 + + allow KNL build without hbwmalloc.h (i.e. emulated) + + we want to be able to run BLIS KNL binaries on non-KNL machines via SDE. + although it is possible to install hbwmalloc implementation on such + systems, it is easier not to, since obviously the performance of SDE + execution is not representative so there is no reason to emulate HBW + allocation. + +commit 17dcd5a33ff91967f67e7c0ba09b4f18754609a4 +Author: Field G. Van Zee +Date: Tue May 2 16:48:43 2017 -0500 + + Fixed stray parentheses in README citations. + +commit 2910d44ff9e1d951d3249313f4ab39d18ea1b48d +Author: Field G. Van Zee +Date: Tue May 2 16:38:43 2017 -0500 + + CHANGELOG update (0.2.2) + +commit 5ca3863220e07972fcefc6682ddd3f6e54fe4a94 +Author: Field G. Van Zee +Date: Tue May 2 15:48:30 2017 -0500 + + Fixed a trsm1m bug that affected right-side cases. + + Details: + - Fixed a bug introduced in 1c732d3 that affected trsm1m_r. The result + was nondeterministic behavior (usually segmentation faults) for certain + problem sizes beyond the 1m instance of kc (e.g. 128 on haswell). The + cause of the bug was my commenting out lines in bli_gemm1m_ukr_ref.c + which explicitly directed the virtual gemm micro-kernel to use temporary + space if the storage preference of the [real domain] gemm ukernel did + not match the storage of the output matrix C. In the context of gemm, + this handling is not needed because agreement between the storage pref + and the matrix is guaranteed by a high-level optimization in BLIS. + However, this optimization is not applied to trsm because the storage + of C is not necessarily the same as the storage of the micro-panels of + B--both of which are updated by the micro-kernel during a trsm + operation. Thus, the guarantee of storage/preference agreement is not + in place for trsm, which means we must handle that case within the + virtual gemm micro-kernel. + - Comment updates and a minor macro change to bli_trsm*_cntx_init() for + 3m1, 4m1a, and 1m. + +commit 1af0b09f5c275ee7bac896cc6f36f42af721d9b5 +Author: Field G. Van Zee +Date: Tue May 2 12:09:39 2017 -0500 + + README.md update. + + Details: + - Updated bibtex entries for 4th BLIS paper, and adds entries for 5th + and 6th BLIS papers. + +commit db4a0bb8ba7cd697d68be8e5632371ee3e59fd63 +Author: Field G. Van Zee +Date: Fri Mar 17 12:07:27 2017 -0500 + + Whitespace reformatting to armv8a kernels file. + + Details: + - Updated formatting of function signature/header in + kernels/armv8a/3/bli_gemm_opt_4x4.c. + +commit e3eb01f6b990e205b15edcbaffd3d54b3ddd1ca4 +Author: Field G. Van Zee +Date: Tue Feb 21 15:33:39 2017 -0600 + + Disabled experiment-related 1m code. + + Details: + - Commented out code in frame/ind/oapi/bli_l3_3m4m1m_oapi.c that was + specifically inserted to facilitate the benchmarking of 1m block-panel + and panel-block algorithms. + - Updates to test/3m4m/Makefile, runme.sh script, and test_gemm.c to + reflect changes used/needed during benchmarking. + +commit 4f61528d56eed6a139eeac9db0c44e56f2d2d136 +Author: Field G. Van Zee +Date: Wed Jan 25 16:25:46 2017 -0600 + + Added 1m-specific APIs for bp, pb gemm algorithms. + + Details: + - Defined bli_gemmbp_cntl_create(), bli_gemmpb_cntl_create(), with the + body of bli_gemm_cntl_create() replaced with a call to the former. + - Defined bli_cntl_free_w_thrinfo(), bli_cntl_free_wo_thrinfo(). Now, + bli_cntl_free() can check if the thread parameter is NULL, and if so, + call the latter, and otherwise call the former. + - Defined bli_gemm1mbp_cntx_init(), bli_gemm1mpb_cntx_init(), both in + terms of bli_gemm1mxx_cntx_init(), which behaves the same as + bli_gemm1m_cntx_init() did before, except that an extra bool parameter + (is_pb) is used to support both bp and pb algorithms (including to + support the anti-preference field described below). + - Added support for "anti-preference" in context. The anti_pref field, + when true, will toggle the boolean return value of routines such as + bli_cntx_l3_ukr_eff_prefers_storage_of(), which has the net effect of + causing BLIS to transpose the operation to achieve disagreement (rather + than agreement) between the storage of C and the micro-kernel output + preference. This disagreement is needed for panel-block implementations, + since they induce a transposition of the suboperation immediately before + the macro-kernel is called, which changes the apparent storage of C. For + now, anti-preference is used only with the pb algorithm for 1m (and not + with any other non-1m implementation). + - Defined new functions, + bli_cntx_l3_ukr_eff_prefers_storage_of() + bli_cntx_l3_ukr_eff_dislikes_storage_of() + bli_cntx_l3_nat_ukr_eff_prefers_storage_of() + bli_cntx_l3_nat_ukr_eff_dislikes_storage_of() + which are identical to their non-"eff" (effectively) counterparts except + that they take the anti-preference field of the context into account. + - Explicitly initialize the anti-pref field to FALSE in + bli_gks_cntx_set_l3_nat_ukr_prefs(). + - Added bli_gemm_ker_var1.c, which implements a panel-block macro-kernel + in terms of the existing block-panel macro-kernel _ker_var2(). This + technique requires inducing transposes on all operands and swapping + the A and B. + - Changed bli_obj_induce_trans() macro so that pack-related fields are + also changed to reflect the induced transposition. + - Added a temporary hack to bli_l3_3m4m1m_oapi.c that allows us to easily + specify the 1m algorithm (block-panel or panel-block). + - Renamed the following cntx_t-related macros: + bli_cntx_get_pack_schema_a() -> bli_cntx_get_pack_schema_a_block() + bli_cntx_get_pack_schema_b() -> bli_cntx_get_pack_schema_b_panel() + bli_cntx_get_pack_schema_c() -> bli_cntx_get_pack_schema_c_panel() + and updated all instantiations. Also updated the field names in the + cntx_t struct. + - Comment updates. + +commit 1d728ccb2394e77365e7c42683db6579c5fba014 +Author: Field G. Van Zee +Date: Fri Nov 25 18:29:49 2016 -0600 + + Implemented the 1m method. + + Details: + - Implemented the 1m method for inducing complex domain matrix + multiplication. 1m support has been added to all level-3 operations, + including trsm, and is now the default induced method when native + complex domain gemm microkernels are omitted from the configuration. + - Updated _cntx_init() operations to take a datatype parameter. This was + needed for the corresponding function for 1m (because 1m requires us + to choose between column-oriented or row-oriented execution, which + requires us to query the context for the storage preference of the + gemm microkernel, which requires knowing the datatype) but I decided + that it made sense for consistency to add the parameter to all other + cntx initialization functions as well, even though those functions + don't use the parameter. + - Updated bli_cntx_set_blkszs() and bli_gks_cntx_set_blkszs() to take + a second scalar for each blocksize entry. The semantic meaning of the + two scalars now is that the first will scale the default blocksize + while the second will scale the maximum blocksize. This allows scaling + the two independently, and was needed to support 1m, which requires + scaling for a register blocksize but not the register storage + blocksize (ie: "packdim") analogue. + - Deprecated bli_blksz_reduce_dt_to() and defined two new functions, + bli_blksz_reduce_def_to() and bli_blksz_reduce_max_to(), for reducing + default and maximum blocksizes to some desired blocksize multiple. + These functions are needed in the updated definitions of + bli_cntx_set_blkszs() and bli_gks_cntx_set_blkszs(). + - Added support for the 1e and 1r packing schemas to packm, including + 1e/1r packing kernels. + - Added a minor optimization to bli_gemm_ker_var2() that allows, under + certain circumstances (specifically, real domain beta and row- or + column-stored matrix C), the real domain macrokernel and microkernel + to be called directly, rather than using the virtual microkernel + via the complex domain macrokernel, which carries a slight additional + amount of overhead. + - Added 1m support to the testsuite. + - Added 1m support to Makefile and runme.sh in test/3m4m. Also simplified + some code in test_gemm.c driver. + +commit 0d1b90286e29aa8b768e280b5286d92c02ad87a1 +Author: Jeff Hammond +Date: Tue Oct 25 21:15:26 2016 -0700 + + never use libm with Intel compilers + + Intel compilers include a highly optimized math library (libimf) that + should be used instead of GNU libm. + + yes, this change is for ALL targets, including those that are not + supported by the Intel compiler. there is no harm in doing this, and it + is future-proof in the event that the Intel compilers support other + architectures. + +commit b150870397e7aee558e61d1bd72a0c0d1d99bee8 +Author: Field G. Van Zee +Date: Fri Dec 8 16:08:41 2017 -0600 + + Removed most "old" directories. + + Details: + - Removed the vast majority of directories named "old", which contained + deprecated code that I wasn't quite ready to jettison from the source + tree. + +commit 270c65985df849297ba1951aa3b56c03948d7775 +Author: Field G. Van Zee +Date: Fri Dec 8 15:21:18 2017 -0600 + + Modified bli_getopt() for thread-safety. + + Details: + - Changed the interface of bli_getopt() to take a new argument, a getopt_t + struct, that stores the values of optarg, optind, opterr, and optopt, + and updated the implementation accordingly. (Previously, these + variables were assumed to be global.) + - Added a function for initializing a getopt_t struct. + - Changed test_libblis.c--currently the only consumer of bli_getopt()--to + utilize the new getopt_t state object. + +commit ce4d8fabc2e39371f89c12192fb707be82ae021a +Merge: 39be59f2 e05a8dfa +Author: Field G. Van Zee +Date: Thu Dec 7 17:36:44 2017 -0600 + + Merge branch 'master' of github.com:flame/blis + +commit 39be59f2a8470f40475907d9dd52639b8a911a92 +Author: Field G. Van Zee +Date: Thu Dec 7 17:35:20 2017 -0600 + + Replaced several macros with static function APIs. + + Details: + - Reimplemented several sets of get/set-style preprocessor macros with + static functions, including those in the following frame/base headers: + auxinfo, cntl, mbool, mem, membrk, opid, and pool. A few headers in + frame/thread were touched as well: mutex_*, thrcomm, and thrinfo. + +commit e05a8dfa7cc7df41e966c1ad04e51c482b308b23 +Merge: 79507337 4423e33d +Author: dnp +Date: Wed Dec 6 16:45:24 2017 -0600 + + Merge branch 'rt' + +commit 4423e33dc593115cda92c5763d756d7ad1298aa9 +Author: dnp +Date: Wed Dec 6 16:35:03 2017 -0600 + + Adding SKX kernels and configuration. + +commit 79507337e140daec7639f6eb3ed9cfe6e123d342 +Author: Field G. Van Zee +Date: Wed Dec 6 16:21:35 2017 -0600 + + Various checks to ensure that arch_t id is in range. + + Details: + - Expanded checking of the arch_t id in bli_gks.c--either passed in from + the caller or as returned from bli_arch_query_id()--against the expected + range of id values. Thanks to Devangi Parikh for suggesting these + additional sanity checks. + +commit fde7c1126c58373ecde83471890b257399144876 +Author: Field G. Van Zee +Date: Mon Dec 4 16:11:01 2017 -0600 + + Added 'uninstall-old-headers' target to Makefile. + + Details: + - Defined a new 'uninstall-old-headers' target that allows users of BLIS to + uninstall no-longer-needed headers left over from previous installations. + - Fixed the 'uninstall-old' target so that it will install both .a and .so + libraries. + - Renamed 'uninstall-old' to 'uninstall-old-libs'. + - Added 'uninstall-old' target (different from previous 'uninstall-old' + target) that combines 'uninstall-old-libs' and 'uninstall-old-headers'. + +commit d4ee770bde213a87aa6049245145318324dc6b51 +Author: Field G. Van Zee +Date: Mon Dec 4 14:53:43 2017 -0600 + + Create/install monolithic cblas.h. + + Details: + - When CBLAS is enabled at configure-time, BLIS now creates a monolithic + cblas.h using the same flatten-header.sh script that was recently + introduced for creating monolithic blis.h header files. The top-level + Makefile will also install this cblas.h file into the install prefix + alongside blis.h when the 'install' target is invoked. The two header + files are compatible with one another. Regardless whether the user's + source #includes cblas.h, both blis.h and cblas.h, or just blis.h, + the user will get the CBLAS function prototypes and enums, as expected. + +commit 52f9e6f1b6468785af8947317656445d4729fc8b +Merge: ab57b979 21360dd8 +Author: Field G. Van Zee +Date: Fri Dec 1 12:28:09 2017 -0600 + + Merge branch 'rt' + +commit 21360dd8e2c7287100645e109acaabcc6ba1140c +Author: Field G. Van Zee +Date: Wed Nov 29 14:11:34 2017 -0600 + + Fixed cntx_t packm query when ker_id > _NUM_PACKM_KERS. + + Details: + - Fixed a subtle bug in bli_cntx_get_[un]packm_ker_dt() in which the + function fails to return NULL when passed a kernel id argument that is + equal to or beyond BLIS_NUM_[UN]PACKM_KERS. Instead, the function was + attempting to index into the cntx_t's packm kernel array, which resulted + in undefined behvaior. Thanks to Devangi Parikh for finding this bug. + +commit 244a6f4e66e8ff091e995f8090ce779c1928aa8b +Author: Field G. Van Zee +Date: Tue Nov 28 17:48:48 2017 -0600 + + Fixed POSIX sed non-compliance in flatten-header.sh. + + Details: + - Changed GNU usage of 'i' and 'a' sed commands used in flatten-header.sh + to POSIX-compliant usage that will work on OS X's sed. + +commit 45078621676833e53a2878af8f89479c4f93b8ab +Author: Field G. Van Zee +Date: Tue Nov 28 15:16:22 2017 -0600 + + Generate/compile with/install monolithic blis.h. + + Details: + - Rewrote monolithify-header.sh (and renamed to flatten-header.sh) so that + headers are inserted recursively. This improves performance by a factor + of 3-4x. + - Modified configure to create an 'include/' directory in which + make can create a monolithic header. + - Modified the top-level Makefile so that a monolithic header is generated + unconditionally prior to compilation (stored in include/) and + so that the single header is installed instead of the 450 or so header + files that reside throughout the framework source tree. + - Added "include/*/*.h" to .gitignore file. + - Removed some pnacl/emscripten leftovers that I intended to include in + a1caeba (mostly in testsuite/Makefile). + - Trivial comment changes to frame/include/bli_f2c.h. + +commit 1f30b1301bf6d6047ec29e57a5fde8eb1072a0ee +Author: Field G. Van Zee +Date: Sat Nov 25 16:54:26 2017 -0600 + + Added missing framework support for x86_64 family. + + Details: + - Added support for the x86_64 configuration family to bli_arch.c and + bli_arch_config.h. Thanks to Johannes Dieterich for reporting this + issue. + - Bumped the default value for BLIS_SIMD_NUM_REGISTERS from 16 to 32 and + the default value for BLIS_SIMD_SIZE from 32 to 64. This will support + configuration families that include Skylake and newer processors without + any supported needed in the bli_family_*.h file. The semantics of these + values have always been "maximum" and not exact values; comments in + bli_kernel_macro_defs.h and the github wiki have been adjusted + accordingly. + +commit 9f39806c4ed484c9ed13edf96005838d977722a9 +Author: Field G. Van Zee +Date: Tue Nov 21 16:03:56 2017 -0600 + + Fixed a bug in e31f0b3/b131b9a. + + Details: + - Erroneously placed the "don't overwrite existing blocksize" logic in + bli_blksz_init*() rather than in bli_cntx_set_blkszs(). It belongs in + the latter because that function copies blocksizes as-is from the + blksz_t function argument to the appropriate field in the cntx_t. If + the blksz_t was previously initialized selectively, based on the sign + of the blocksize value passed into bli_blksz_init*(), that just leaves + some fields possibly uninitialized (with garbage values), which + definitely will not work. + - The aforementioned logic has been moved to bli_cntx_set_blkszs() via + a new function bli_blksz_copy_if_pos(), which selectively copies only + the blocksizes that are greater than zero. + +commit b131b9a025c15f548d4c2952a9ec85eee3d139b1 +Author: Field G. Van Zee +Date: Tue Nov 21 14:30:26 2017 -0600 + + Updated configs to omit setting some blocksizes. + + Details: + - Employ the new semantics of bli_blksz_init*() in e31f0b3 in various + sub-configurations' bli_cntx_init_*() functions by passing in 0 for + register and cache blocksizes that correpond to gemm microkernel + datatypes that were not registered, allowing the default values + set by the bli_cntx_init_*_ref() function call to remain. + +commit 499a4c002f895744ecaf81ef7f62d2d6d0d7d594 +Merge: e31f0b3e 6c3ba502 +Author: Field G. Van Zee +Date: Tue Nov 21 14:25:08 2017 -0600 + + Merge branch 'rt' of github.com:flame/blis into rt + +commit e31f0b3e2dba19ca8a2946bc21beb136a42d0f57 +Author: Field G. Van Zee +Date: Tue Nov 21 14:21:25 2017 -0600 + + Subtle update to bli_blksz_init*() API. + + Details: + - Updated the semantics of bli_blksz_init() and bli_blksz_init_ed() so + that non-positive blocksize values are ignored entirely. This provides + an easy way to indicate that certain existing values should not be + touched by the update. Thanks to Devangi Parikh for feedback that led + to these changes. + +commit 6c3ba502a11f87bc67555d26154cfd39d0af1bac +Author: Field G. Van Zee +Date: Tue Nov 21 13:50:53 2017 -0600 + + Added 'x86_64' sub-config directory. + + Details: + - Added missing x86_64 configuration directory, which was intended to be + part of b7ca580. + - Added -Wfatal-errors compiler warning flag to all configurations so that + compilation stops after the first error. + - Changed the vectorization flags for intel64 configuration to be compatible + with 'penryn', the oldest sub-config included in that family. + - Changed the vectorization flags for penryn to target the 'core2' + microarchitecture and ssse3. + +commit 25eee3cc49b0631812485d4d5ceef0c23ed1b6dd +Author: Field G. Van Zee +Date: Tue Nov 21 12:34:20 2017 -0600 + + Added a dummy file to kernels/generic. + + Details: + - Added a dummy file to kernels/generic, which was previously empty, so + that git would begin tracking the otherwise-empty directory. This + directory's existence is necessary for proper execution of configure + for any configuration family that contains the 'generic' + sub-configuration. Thanks to Johannes Dieterich for reporting the + issue that led to this fix. + +commit ef024ce4cafa217669eaabb31ff8ab6df93cca05 +Author: Field G. Van Zee +Date: Mon Nov 20 18:08:29 2017 -0600 + + More tweaks to monolithify-header.sh + + Details: + - Further fixes monolithify-header.sh script. + - Removed unnecessary #include "blis.h" from frame/3/bli_l3_packm.h. + +commit 5028e7dec269b62895511453272585da36e591b5 +Author: Field G. Van Zee +Date: Mon Nov 20 17:00:37 2017 -0600 + + Second attempt to implement travis_wait. + + Details: + - Corrected accidental misplacement of the travis_wait prefix (on the + wrong line of the .travis.yml file) in commit 13e5d91. + +commit 13e5d9107b3763cba46fb1bae87476852601b47c +Author: Field G. Van Zee +Date: Mon Nov 20 15:57:06 2017 -0600 + + Added travis_wait prefix to testsuite via Travis. + + Details: + - It appears that Travis CL has implemented a new policy that results in + a test failing if it does not produce any output for more than 10 + minutes. (Two test instances are now failing in Travis despite the most + recent commit not affecting the library or testsuite.) This issue can + be worked around by executing the test run via travis_wait, which takes + an optional time parameter. This commit attempts to use 'travis_wait 30' + in the .travis.yml file to prevent the early failure at 10 minutes. + +commit a1caeba0ea79c8fecb1abadca1f91c6367ab3afb +Author: Field G. Van Zee +Date: Mon Nov 20 13:31:20 2017 -0600 + + Removed pnacl, emscripten support from Makefile. + +commit 78199c539beaa50f37893add220261ce0dcb921a +Merge: b3d8ab2e ab57b979 +Author: praveeng +Date: Mon Nov 20 15:51:20 2017 +0530 + + Merge master code till 01-Nov-2017 to amd-staging + + Change-Id: I40b53f876db84c8b947b3f2385c9b882245c6603 + +commit 9df6dda9ec51a0d40166169d2d8a2f84b42266e6 +Author: Field G. Van Zee +Date: Sat Nov 18 19:03:26 2017 -0600 + + Improvements, bugfixes to monolithify-header.sh. + +commit 21d26201f90b884eb8d5de279ed74bbd244ffcb5 +Merge: 43baa3b3 b7ca5806 +Author: Field G. Van Zee +Date: Sat Nov 18 14:16:53 2017 -0600 + + Merge branch 'rt' of github.com:flame/blis into rt + +commit 43baa3b327d5ae1e2ba619432687b4dd849b05e3 +Author: Field G. Van Zee +Date: Sat Nov 18 14:14:44 2017 -0600 + + Removed unnecessary flags for generic config. + + Details: + - Removed -D_POSIX_C_SOURCE=200112L and -m64 flags from make_defs.mk file + of generic sub-configuration. These flags are generally not necessary, + and particularly not desirable for the generic configuration since they + unnecessarily restrict the environments in which the configuration can + be built. + +commit b7ca580618f9382b7982168fd035ed058f83e4c2 +Author: iotamudelta +Date: Sat Nov 18 14:56:05 2017 -0500 + + [WIP] Add x86 and x86_64 processor families. (#154) + + * Add x86 and x86_64 processor families. + * Use generic config as fallback for more families. + + After discussion with fgvanzee, a) it's "generic" and 2) use it for all the families as a fallback. Goal is that if a specific CPU is not yet supported by a family (say a new Intel microarchitecture on x86_64), it'll fall through to still work with the slower "generic" kernels + +commit 870597d1663aaba1b74d7654b1d4946280aa0d3f +Author: Field G. Van Zee +Date: Fri Nov 17 17:06:42 2017 -0600 + + Added bash script for creating monolithic headers. + + Details: + - Added a new script, monolithify-header.sh, to the 'build' directory. + This script recursively replaces all #include directives in a selected + file with the contents of the header files referenced by each directive. + The idea is to "flatten" a tree of .h files into a single file, with + the script acting as a C preprocessor that only processes #include + directives. + +commit c76f77f4cc1e71988251c5e63cf6ef137477bf9c +Author: Field G. Van Zee +Date: Fri Nov 17 15:10:52 2017 -0600 + + Removed unnecessary #include "blis.h" from header. + + Details: + - Removed an errant #include "blis.h directive from bli_cntx_ind_stage.h. + The generaly policy is that no header file in BLIS should include + blis.h. This will be important in the near future when using a tool to + recursively create a monolithic blis.h file from its consitutent + headers. + +commit 2bb9bc6e9536fa239fbc19a7efaaf151116e15b4 +Author: Field G. Van Zee +Date: Fri Nov 17 13:50:14 2017 -0600 + + Miscellaneous tweaks to gks, rt functionality. + + Details: + - Updated bli_cpuid_query_id() so that BLIS_ARCH_GENERIC is always returned + if the hardware fails to test positive for any supported sub-configuration. + - Defined bli_gks_init_ref_cntx(), which will call the context initialization + function bli_cntx_init_configname() for the sub-configuration 'configname' + associated with the arch_t id returned by bli_arch_query_id(). This makes + initializing a reference context easy for experts who wish to construct + those contexts. + +commit b3d8ab2ea02c127ab241532abc214624f35bfaab +Merge: 189ffbb0 fe71c06e +Author: Santanu Thangaraj +Date: Wed Nov 15 01:33:12 2017 -0500 + + Merge "Added AMD copyright line to the changed files in last 3 commits" into amd-staging + +commit fe71c06e42b072407c83112779055b0afb67173d +Author: Nisanth M P +Date: Wed Nov 15 11:11:17 2017 +0530 + + Added AMD copyright line to the changed files in last 3 commits + + Change-Id: I37d5dbbbe1b199e07529610a5e9cc9e49d067c66 + +commit d5bf79e50bf97072bbe7117c86b7c45e6e707ea0 +Author: Field G. Van Zee +Date: Mon Nov 13 14:24:29 2017 -0600 + + Miscellaneous tweaks and fixes. + + Details: + - Fixed incorrect calling sequence in bli_cntx_init_knl.c--an instance of + bli_blksz_init_easy() that should have been bli_blksz_init(). + - Fixed a bug in code that is supposed to output the list of sub-directories + in the 'config' directory when configure script is run with no arguments. + - Expanded the output of "make showconfig" to include more info from config.mk. + - Minor changes to build/auto-detect/cpuid_x86.c, mostly in preparation for + someone to add excavator and zen support. + - Added a link to the ConfigurationHowTo wiki to config_registry. + - Other minor tweaks to configure. + +commit 673e5184030532c4ebd9fdeecbaa6442bb3ad54f +Merge: 2c51356a 8f150f28 +Author: Field G. Van Zee +Date: Wed Nov 1 17:37:42 2017 -0500 + + Merge branch 'rt' of github.com:flame/blis into rt + +commit 2c51356a8b2699c99f9507c80d69c08a35d45fe3 +Author: Field G. Van Zee +Date: Wed Nov 1 17:37:02 2017 -0500 + + Implemented runtime hardware detection via cpuid. + + Details: + - Added runtime support for selecting an appropriate arch_t value based + on the results of the cpuid instruction (for x86_64). This allows + deferral of choosing a context (kernels, blocksizes, etc.) until + runtime, which allows BLIS to be built with support for multiple + microarchitectures. Currently, only amd64 and intel64 configurations + are registered in the config_registry; however, one could create + custom configuration families to support arbitrary sets of x86_64 + microarchitectures. + - Current Intel microarchitectures supported via cpuid are knl, haswell, + sandybridge, and penryn. + - Current AMD microarchitectures supported via cpuid are: zen, excavator, + steamroller, piledriver, and bulldozer. + +commit ab57b979046479bcda7f83165838a80117c2ad95 +Author: Field G. Van Zee +Date: Wed Nov 1 11:51:41 2017 -0500 + + Revert to default SIMD alignment for bulldozer. + + Details: + - Removed the default-overriding #define of BLIS_SIMD_ALIGN_SIZE set in + config/bulldozer/bli_kernel.h. Not sure where this value came from, but + it would seem to allow for insufficient starting address alignment for + any matrices created via bli_malloc_user(), such as via + bli_obj_create(). Thanks to Rene Sitt for reporting the behavior that + led us to this bug. + - This commit is a manual patch of the same fix made to the 'rt' branch + in 8f150f2. + +commit 8f150f28a678c4a0c1591400177ad7cca81fcaec +Author: Field G. Van Zee +Date: Wed Nov 1 11:41:45 2017 -0500 + + Revert to default SIMD alignment for bulldozer. + + Details: + - Removed the default-overriding #define of BLIS_SIMD_ALIGN_SIZE set in + bli_family_bulldozer.h. Not sure where this value came from, but it + would seem to allow for insufficient starting address alignment for + any matrices created via bli_malloc_user(), such as via + bli_obj_create(). Thanks to Rene Sitt for reporting the behavior that + led us to this bug. + +commit e3f10557caf114441fbfff990e3ce3576c177bdc +Author: Field G. Van Zee +Date: Mon Oct 30 13:37:54 2017 -0500 + + Use perl for some substitution for OS X compatibility. + + Details: + - Discovered that sed commands where the replacement string contains '\n' + are problematic with the version of sed present in OS X. For these cases + cases in the configure script, we instead use 'perl -pe' for + search-and-replace functionality. + - Various other minor comment/whitespace tweaks to configure. + - Removed remaining lines of code related to setting/checking variables to + track "unregistered" configurations. + +commit dd45cfdfc3d8f9acf4cf7f69138d9b83dafc8842 +Merge: 3e4f42a4 f60c827b +Author: Field G. Van Zee +Date: Mon Oct 30 12:23:05 2017 -0500 + + Merge branch 'master' into rt + +commit f60c827ba95f452c8454fb914f5564f4895bf644 +Author: Devin Matthews +Date: Mon Oct 30 10:04:42 2017 -0500 + + Fix CVECFLAGS for bulldozer config. + +commit 3e4f42a4d2ebb37b95988933d92e561c5b2cc201 +Author: Field G. Van Zee +Date: Fri Oct 27 11:41:37 2017 -0500 + + Typecast l1mkr_t enum value prior to comparison. + + Details: + - Typecast l1mkr_t enum value in bli_cntx.h to guint_t before testing for + out-of-range value. This is an attempt to pacify a strange warning from + clang on OS X that is seemingly the result of the following compiler + warning flag: + -Wtautological-constant-out-of-range-compare + +commit aec6e038d942d35b81bbd723a640cce2c054fb8e +Author: Field G. Van Zee +Date: Thu Oct 26 16:12:36 2017 -0500 + + Removed associative arrays from configure. + + Details: + - Implemented a replacement for associative arrays in the configure script + that does not utilize arrays, and therefore works in pre-4.0 versions of + bash. (It appears that Mac OS X will be stuck with version 3.2 indefinitely + due to bash switching to the GPL 3.0 license starting with version 4.0.) + +commit 189ffbb0d37262b21acddc0d35b4a22f2cbbca94 +Merge: 06e0e635 3eb44f67 +Author: Santanu Thangaraj +Date: Wed Oct 25 02:00:30 2017 -0400 + + Merge changes Ie115b206,I7ce6cfa2,Iff59b6f4 into amd-staging + + * changes: + Adding __attribute__((constructor/destructor)) for CLANG case. + Thread Safety: Move bli_init() before and bli_finalize() after main() + Thread safety: Make the global induced method status array local to thread + +commit 3eb44f67618b91ae5f5f0aaaba67e38f16042ee4 +Author: Nisanth M P +Date: Tue Oct 24 16:36:36 2017 +0530 + + Adding __attribute__((constructor/destructor)) for CLANG case. + + CLANG supports __attribute__, but its documentation doesn't + mention support for constructor/destructor. Compiling with + clang and testing shows that it does support this. + + Change-Id: Ie115b20634c26bda475cc09c20960d687fb7050b + +commit 07c352188bf5265af242255f8e6fcb97050d973d +Author: Field G. Van Zee +Date: Mon Oct 23 16:59:22 2017 -0500 + + Added "generic" configuration. + + Details: + - Added a "generic" configuration that leaves the default blocksizes and + kernels unchanged. This replaces the older "reference" configuration. + Updated auto-detect script and code accordingly. + - Added support for generic configuration to arch_t (bli_type_defs.h), + bli_gks_init() (bli_gks.c), and bli_arch_config.h + - Moved bli_arch_query_id() to bli_arch.c (and prototype to bli_arch.h). + - Whitespace changes to configurations' make_defs.mk files. + +commit c1a98d6f70608b02a1e6bcad6ba020a60773dace +Author: Field G. Van Zee +Date: Mon Oct 23 14:24:41 2017 -0500 + + Minor update to .travis.yml file. + +commit 75b9383f01caa8b83f8be0117e15085b0d807ba6 +Author: Field G. Van Zee +Date: Fri Oct 20 16:41:22 2017 -0500 + + Minor header renaming ahead of bli_arch.c. + + Details: + - Renamed the various configurations' "bli_arch_.h" header files + (replacing "arch" with "family") to free up the 'bli_arch' namespace for a + different purpose (hardware detection). + - Renamed "bli_arch.h" and "bli_arch_pre_macro_defs.h" in frame/include to + "bli_arch_config.h" and "bli_arch_config_pre.h", respectively. + +commit 482af51add26d5ed103c3e3f167657f273b32c7a +Author: Field G. Van Zee +Date: Fri Oct 20 15:44:26 2017 -0500 + + Fixed 'make test' target from top-level Makefile. + + Details: + - Updated the top-level Makefile's build rule for testsuite object files to + properly obtain CFLAGS via get-frame-cflags-for() function instead of + simply using the $(CFLAGS) variable (which is empty). This means that + 'make test' should now work as expected. + +commit 3c269f700d207efe6c04193f09d519c88c1d4045 +Author: Field G. Van Zee +Date: Fri Oct 20 13:57:21 2017 -0500 + + Makefile updates for test drivers, testsuite. + + Details: + - Fixed semi-broken testsuite Makefile and very-broken test driver Makefiles, + as well as those for test/3m4m, test/thread_ranges, and test/exec_sizes + sub-directories. + - Factored out much of the top-level Makefile into common.mk. A Makefile + needs only set DIST_PATH to the relative path to the top level of the + BLIS source distribution before including common.mk in order to acquire + all of the definitions typically needed in a Makefile that tests BLIS. + +commit 0557189d463446b4c32077cdcf0467fa71ca68dc +Author: Field G. Van Zee +Date: Wed Oct 18 15:05:27 2017 -0500 + + Minor updates to .travis.yml, configure script. + +commit 2553734d1d62043793f4e783a027349ef6d4d563 +Merge: 453deb29 37534279 +Author: Field G. Van Zee +Date: Wed Oct 18 13:46:50 2017 -0500 + + Merge branch 'master' into rt + +commit 375342799cbae981c28d831793af588d7951f3f6 +Author: Field G. Van Zee +Date: Wed Oct 18 13:41:25 2017 -0500 + + Removed a duplicate bli_avx512_macros.h header. + + Details: + - Removed a duplicate header file that was causing problems during + installation for the 'knl' configuration. Thanks to Victor Eijkhout + for reporting this issue. + +commit 453deb29068889698e274f269c9aa90eea99b527 +Author: Field G. Van Zee +Date: Wed Oct 18 13:29:32 2017 -0500 + + Implemented runtime kernel management. + + Details: + - Reworked the build system around a configuration registry file, named + config_registry', that identifies valid configuration targets, their + constituent sub-configurations, and the kernel sets that are needed by + those sub-configurations. The build system now facilitates the building + of a single library that can contains kernels and cache/register + blocksizes for multiple configurations (microarchitectures). Reference + kernels are also built on a per-configuration basis. + - Updated the Makefile to use new variables set by configure via the + config.mk.in template, such as CONFIG_LIST, KERNEL_LIST, and KCONFIG_MAP, + in determining which sub-configurations (CONFIG_LIST) and kernel sets + (KERNEL_LIST) are included in the library, and which make_defs.mk files' + CFLAGS (KCONFIG_MAP) are used when compiling kernels. + - Reorganized 'kernels' directory into a "flat" structure. Renamed kernel + functions into a standard format that includes the kernel set name + (e.g. 'haswell'). Created a "bli_kernels_.h" file in each + kernels sub-directory. These files exist to provide prototypes for the + kernels present in those directories. + - Reorganized reference kernels into a top-level 'ref_kernels' directory. + This directory includes a new source file, bli_cntx_ref.c (compiled on + a per-configuration basis), that defines the code needed to initialize + a reference context and a context for induced methods for the + microarchitecture in question. + - Rewrote make_defs.mk files in each configuration so that the compiler + variables (e.g. CFLAGS) are "stored" (renamed) on a per-configuration + basis. + - Modified bli_config.h.in template so that bli_config.h is generated with + #defines for the config (family) name, the sub-configurations that are + associated with the family, and the kernel sets needed by those + sub-configurations. + - Deprecated all kernel-related information in bli_kernel.h and transferred + what remains to new header files named "bli_arch_.h", which + are conditionally #included from a new header bli_arch.h. These files + are still needed to set library-wide parameters such as custom + malloc()/free() functions or SIMD alignment values. + - Added bli_cntx_init_.c files to each configuration directory. + The files contain a function, named the same as the file, that initializes + a "native" context for a particular configuration (microarchitecture). The + idea is that optimized kernels, if available, will be initialized into + these contexts. Other fields will retain pointers to reference functions, + which will be compiled on a per-configuration basis. These bli_cntx_init_*() + functions will be called during the initialization of the global kernel + structure. They are thought of as initializing for "native" execution, but + they also form the basis for contexts that use induced methods. These + functions are prototyped, along with their _ref() and _ind() brethren, by + prototype-generating macros in bli_arch.h. + - Added a new typedef enum in bli_type_defs.h to define an arch_t, which + identifies the various sub-configurations. + - Redesigned the global kernel structure (gks) around a 2D array of cntx_t + structures (pointers to cntx_t, actually). The first dimension is indexed + over arch_t and the inner dimension is the ind_t (induced method) for + each microarchitecture. When a microarchitecture (configuration) is + "registered" at init-time, the inner array for that configuration in the + 2D array is initialized (and allocated, if it hasn't been already). The + cntx_t slot for BLIS_NAT is initialized immediately and those for other + induced method types are initialized and cached on-demand, as needed. At + cntx_t registration, we also store function pointers to cntx_init functions + that will initialize (a) "reference" contexts and (b) contexts for use with + induced methods. We don't cache the full contexts for reference contexts + since they are rarely needed. The functions that initialize these two kinds + of contexts are generated automatically for each targeted sub-configuration + from cpp-templatized code at compile-time. Induced method contexts that + need "stage" adjustments can still obtain them via functions in + bli_cntx_ind_stage.c. + - Added new functions and functionality to bli_cntx.c, such as for setting + the level-1f, level-1v, and packm kernels, and for converting a native + context into one for executing an induced method. + - Moved the checking of register/cache blocksize consistency from being cpp + macros in bli_kernel_macro_defs.h to being runtime checks defined in + bli_check.c and called from bli_gks_register_cntx() at the time that the + global kernel structure's internal context is initialized for a given + microarchitecture/configuration. + - Deprecated all of the old per-operation bli_*_cntx.c files and removed + the previous operation-level cntx_t_init()/_finalize() invocations. + Instead, we now query the gks for a suitable context, usually via + bli_gks_query_cntx(). + - Deprecated support for the 3m2 and 3m3 induced methods. (They required + hackery that I was no longer willing to support.) + - Consolidated the 1e and 1r packm kernels for any given register blocksize + into a single kernel that will branch on the schema and support packing + to both formats. + - Added the cntx_t* argument to all packm kernel signatures. + - Deprecated the local function pointer array in all bli_packm_cxk*.c files + and instead obtain the packm kernel from the cntx_t. + - Added bli_calloc_intl(), which serves as the calloc-equivalent to to + bli_malloc_intl(). Useful when we wish to allocate and initialize to + zero/NULL. + - Converted existing cpp macro functions defined in bli_blksz.h, bli_func.h, + bli_cntx.h into static functions. + +commit 4607aac297e55ad540cbe5fffbe02e6b1889c181 +Author: Nisanth M P +Date: Mon Oct 16 22:06:57 2017 +0530 + + Thread Safety: Move bli_init() before and bli_finalize() after main() + + BLIS provides APIs to initialize and finalize its global context. + One application thread can finalize BLIS, while other threads + in the application are stil using BLIS. + + This issue can be solved by removing bli_finalize() from API. + One way to do this is by getting bli_finalize() to execute by default + after application exits from main(). + + GCC supports this behaviour with the help of __attribute__((destructor)) + added to the function that need to be executed after main exits. + + Similarly bli_init() can be made to run before application enters main() + so that application need not call it. + + Change-Id: I7ce6cfa28b384e92c0bdf772f3baea373fd9feac + +commit 0f5ce26fc597cda6e8ae93a7526f52eb8cba01e9 +Author: Nisanth M P +Date: Mon Oct 16 21:07:50 2017 +0530 + + Thread safety: Make the global induced method status array local to thread + + BLIS retains a global status array for induced methods, and provides + APIs to modify this state during runtime. So, one application thread + can modify the state, before another starts the corresponding + BLIS operation. + + This patch solves this issue by making the induced method status array + local to threads. + + Change-Id: Iff59b6f473771344054c010b4eda51b7aa4317fe + +commit b882648af87deb1b365fc6b3e94151e69c5ccfa4 +Merge: 8b379069 e02d3cb8 +Author: Field G. Van Zee +Date: Wed Oct 11 16:32:21 2017 -0500 + + Merge branch 'master' into rt + +commit 06e0e6351acb9481225975ad9a4e0b8925336621 +Author: sthangar +Date: Thu Sep 28 12:15:36 2017 +0530 + + The inner loop paralleization is turned off by default, the JR and IR loop parameters are set to 1 by default + + Change-Id: I8c3c2ecbbd636259f6ffb92768ec04148205c3e5 + +commit e02d3cb84190a345ebe9b32f53db03a1838976b1 +Author: Field G. Van Zee +Date: Tue Sep 26 19:02:53 2017 -0500 + + Fixed a pthread typo in previous commit. + + Details: + - Misnamed 'pthread_mutex_t' type in bli_memsys.c as 'thread_mutex_t'. + +commit f5962a1aae0fb3c9be104d0035c0d73210e7f670 +Author: Field G. Van Zee +Date: Tue Sep 26 17:00:04 2017 -0500 + + Fixed bugs in gemm/gemmtrsm ukr tests in testsuite. + + Details: + - Fixed a bug in gemmtrsm test module that was due to improper partitioning + into a k x k triangular matrix for the purposes of obtaining an mr x k + micropanel of A with which to test. + - Fixed a bug in gemm and gemmtrsm test modules that would only manifest for + very large k (depending on the product of mr x kc on that architecture). + The bug arose from the fact that the test module was triggering the + allocation of blocks from the internal memory pools, which are limited in + size. This allocation imposes an implicit assumption that the micro- + panel being tested with will fit inside, and this assumption is violated + for large values of k. Arbitrarily large k may now be tested for both + operation tests. + - Added OpenMP/pthread critical sections around the setting or getting of + statuses from the induced method operation lookup table in bli_l3_ind.c. + - Added the 'static' keyword to all pthread_mutex_t global variables in BLIS. + - Thanks to Nisanth Padinharepatt of AMD for reporting the first and third + issues. + +commit 8e917b256ca2d4bcdc059fe98d86be8775c69561 +Author: Field G. Van Zee +Date: Sat Sep 9 14:10:15 2017 -0500 + + Updated bibtex info for BLIS5 (3m4m) article. + +commit 7be887057358df4978a4833eeae0c17e15acd9d1 +Author: Nisanth M P +Date: Mon Aug 28 17:38:22 2017 +0530 + + Merging "Adding auto hardware detection for Zen" + + Change-Id: Id450fb0c4f91a5cd5cbdc06970f4f9ed28dd8520 + +commit e056d810d16621891ead032603de0c2105cfc0f7 +Author: sthangar +Date: Mon Aug 28 16:44:42 2017 +0530 + + Bug fix for the testsuite build failing + + Change-Id: I7cd8c9d187387c48b2564e45cbfb8df985e93d77 + +commit 83796b7caf745fafc263e9e5e1bfcf5eff00c025 +Merge: 8176f4e4 d1ee7762 +Author: Kiran Varaganti +Date: Mon Aug 28 05:23:28 2017 -0400 + + Merge "Adding auto hardware detection for Zen" into amd-staging + +commit d1ee776202b26874333af7a91b6d2686342c4c81 +Author: sthangar +Date: Wed Aug 23 13:01:14 2017 +0530 + + Adding auto hardware detection for Zen + + Change-Id: I40ce6705dd66b35000c4ccddffad1c5b65998caf + +commit 8176f4e43872714b997f1a5f83056daadb0ff1a5 +Merge: 12413018 adafe974 +Author: praveeng +Date: Mon Aug 28 12:21:16 2017 +0530 + + resolving conflicts bli_gemm_front.c and LICENCE + + Change-Id: Id24ce53896d4c1c7ceccc3e004014a0ecceb5474 + +commit 57e1e5cd51e7ffe8612c96a20b6a041b55426ddb +Merge: f86ce54d d6ef56c6 +Author: Nisanth M P +Date: Tue Aug 22 17:07:44 2017 +0530 + + Merge AMD authored changes + +commit adafe974b4bc3fc0663bc2f6f4ce2fde71a97988 +Merge: f86ce54d 7dc78b49 +Author: Devin Matthews +Date: Tue Aug 15 15:17:21 2017 -0500 + + Merge pull request #150 from devinamatthews/vzeroupper + + Add vzeroupper to Intel AVX kernels. + +commit 7dc78b49f97e6b3cd6d72fcdc588ace534d0e700 +Author: Devin Matthews +Date: Tue Aug 15 10:02:25 2017 -0500 + + Add vzeroupper to Intel AVX kernels. + +commit f86ce54d6f315006984534fe29e47a2deaacc9f5 +Author: Field G. Van Zee +Date: Thu Aug 10 16:24:28 2017 -0500 + + Removed trailing enum commas from bli_type_defs.h. + + Details: + - Removed trailing commas from enums in bli_type_defs.h. Thanks to + Erling Andersen for pointing out this inconsistency and suggesting + the change. + +commit 60a1eeb2317939d732b9eb6ff1e0d6d668c9a1e5 +Author: Field G. Van Zee +Date: Sat Aug 5 13:04:31 2017 -0500 + + Added edge handling to _determine_blocksize_b(). + + Details: + - Added explicit handling of situations where i == dim to + bli_determine_blocksize_b_sub(). This isn't actually needed by any + current use case within BLIS, but handling the situation is nonetheless + prudent. Thanks to Minh Quan for reporting this issue and requesting + the fix. + +commit b01c80829907d50ec79977fba8e7b53cfe7db80a +Author: Field G. Van Zee +Date: Fri Aug 4 14:17:44 2017 -0500 + + Fixed a minor bug in level-3 packm management. + + Details: + - Fixed a bug in bli_l3_packm() that caused cntl_t-cached packed mem_t + entries to be released and then re-acquired unnecessarily. (In essence, + the "<" operands in the conditional that guards the + release-and-reacquire code block simply needed to be swapped.) The bug + should have only affected performance (rather than the computed result). + Thanks to Minh Quan for identifying and reporting the bug. + +commit 8b379069fcd4811669855b1248ece831f190dff6 +Merge: 1f3a5819 05925dd5 +Author: Field G. Van Zee +Date: Tue Aug 1 15:30:40 2017 -0500 + + Merge branch 'master' into rt + +commit 05925dd5d30e8f403bb671ce33029170d65ce7c0 +Merge: 803bbef0 cecdc05d +Author: Devin Matthews +Date: Tue Aug 1 09:31:02 2017 -0500 + + Merge pull request #146 from devinamatthews/master + + Change lsame_ signature to match lapacke. + +commit cecdc05d2834786a84ff85775d3f99a958c0765a +Author: Devin Matthews +Date: Mon Jul 31 15:19:51 2017 -0500 + + Change lsame_ signature to match lapacke. + +commit 803bbef0a386dd0571ad389f69d55154dbfe3c50 +Author: Field G. Van Zee +Date: Sat Jul 29 20:17:05 2017 -0500 + + Fixed pthreads compile bug with previous commit. + + Details: + - Erroneously passed family parameter into l3int_t function despite + that function not taking the parameter. Oops. + +commit c63980f4ca750618f359031d0691289b1abf5146 +Author: Field G. Van Zee +Date: Sat Jul 29 14:53:39 2017 -0500 + + Moved 'family' field from cntx_t to cntl_t. + + Details: + - Removed the family field inside the cntx_t struct and re-added it to the + cntl_t struct. Updated all accessor functions/macros accordingly, as well + as all consumers and intermediaries of the family parameter (such as + bli_l3_thread_decorator(), bli_l3_direct(), and bli_l3_prune_*()). This + change was motivated by the desire to keep the context limited, as much + as possible, to information about the computing environment. (The family + field, by contrast, is a descriptor about the operation being executed.) + - Added additional functions to bli_blksz_*() API. + - Added additional functions to bli_cntx_*() API. + - Minor updates to bli_func.c, bli_mbool.c. + - Removed 'obj' from bli_blksz_*() API names. + - Removed 'obj' from bli_cntx_*() API names. + - Removed 'obj' from bli_cntl_*(), bli_*_cntl_*() API names. Renamed routines + that operate only on a single struct to contain the "_node" suffix to + differentiate with those routines that operate on the entire tree. + - Added enums for packm and unpackm kernels to bli_type_defs.h. + - Removed BLIS_1F and BLIS_VF from bszid_t definition in bli_type_defs.h. + They weren't being used and probably never will be. + +commit 07837395560d413a1ba828163b41186e21a7bcfe +Merge: ca1d1d85 ad8610b4 +Author: Field G. Van Zee +Date: Fri Jul 21 16:49:48 2017 -0500 + + Merge pull request #139 from Maratyszcza/emscripten + + Fix Emscripten builds + +commit ad8610b4415cc7982804d74f9aba29875e9e2b6c +Merge: 8772a0b3 ca1d1d85 +Author: Field G. Van Zee +Date: Fri Jul 21 15:18:33 2017 -0500 + + Merge branch 'master' into emscripten + +commit ca1d1d8560c9ab1a7e3b0ac43ac70d08075bf904 +Merge: b537b5bb 733faf84 +Author: Devin Matthews +Date: Fri Jul 21 09:49:50 2017 -0500 + + Merge pull request #144 from devinamatthews/fix_atomics_on_bgq + + Add fallbacks to __sync_* or __c11_atomic_* builtins... + +commit 733faf848dcc54834fcdfbb0185dc644978d8864 +Author: Devin Matthews +Date: Thu Jul 20 14:50:13 2017 -0500 + + Clang can't make up it's mind what to support. + +commit 7425d0744d9e9cd29a887120e57c2b43ba287040 +Author: Devin Matthews +Date: Thu Jul 20 12:54:58 2017 -0500 + + Add default #define for __has_extension. + +commit b537b5bbe8cbee459a85bac11458498ae2bce4de +Merge: 1f1ec0db 7f41bb0a +Author: Devin Matthews +Date: Thu Jul 20 10:58:39 2017 -0500 + + Merge pull request #133 from devinamatthews/haswell-packdim + + Fix prefetching in haswell ukernel + +commit 8823f91a14638ce6f4e45e67df03212bb61609d6 +Author: Devin Matthews +Date: Thu Jul 20 10:04:34 2017 -0500 + + Add fallbacks to __sync_* or __c11_atomic_* builtins when __atomic_* is not supported. Fixes #143. + +commit 1f1ec0db9380b87679d5c771c4594daa1cfc5f0d +Author: Field G. Van Zee +Date: Wed Jul 19 15:40:48 2017 -0500 + + Updated ar option list used by all configurations. + + Details: + - Dropped 'u' from the list of modifiers passed into the library archiver + ar. Previously, "cru" was used, while now we employ only "cr". This + change was prompted by a warning observed on Ubuntu 16.04: + + ar: `u' modifier ignored since `D' is the default (see `U') + + This caused me to realize that the default mode causes timestamps to be + zero, and thus the 'u' option, which causes only changed object files to + be inserted, is not applicable. + +commit 5caaba2d61cbbc36d63102a0786ece28ff797f72 +Author: Field G. Van Zee +Date: Wed Jul 19 13:51:53 2017 -0500 + + Added --force-version=STRING option to configure. + + Details: + - Added an option to configure that allows the user to force an arbitrary + version string at configure-time. The help text also now describes the + usage information. + - Changed the way the version string is communicated to the Makefile. + Previously, it was read into the VERSION variable from the 'version' file + via $(shell cat ...). Now, the VERSION variable is instead set in + config.mk (via a configure-substituted anchor from config.mk.in). + +commit 13175c5fb70fb6a378d5fff6ecede62e5ea6a1f6 +Author: Field G. Van Zee +Date: Tue Jul 18 17:56:00 2017 -0500 + + Updated openmp/pthread barriers with GNU atomics. + + Details: + - Updated the non-tree openmp and pthreads barriers defined in + bli_thrcomm_openmp.c and bli_thrcomm_pthreads.c to instead call a common + implementation in bli_thrcomm.c, bli_thrcomm_barrier_atomic(). This new + implementation goes through the same motions as the previous codes, but + protects its loads and increments with GNU atomic built-ins. These atomic + statements take memory ordering parameters that allow us to specify just + enough constraints for the barrier to work as intended on weakly-ordered + hardware. The prior implementation was only guaranteed to work on systems + with strongly- ordered memory. (Thanks to Devin Matthews for suggesting + this change and his crash-course in atomics and memory ordering.) + - Removed 'volatile' from structs' barrier field declarations in + bli_thrcomm_*.h. + - Updated bli_thrcomm_pthread.? files to use renamed struct barrier fields + consistent with that of the _openmp.? files. + - Updated other bli_thrcomm_* files to rename "communicator" variables to + simply "comm". + +commit 0e58ba1b3aa84700ca51a96f1c0eed6067562fba +Author: Field G. Van Zee +Date: Mon Jul 17 19:03:22 2017 -0500 + + Added API to set mt environment variables. + + Details: + - Renamed bli_env_get_nway() -> bli_thread_get_env(). + - Added bli_thread_set_env() to allow setting environment variables + pertaining to multithreading, such as BLIS_JC_NT or BLIS_NUM_THREADS. + - Added the following convenience wrapper routines: + bli_thread_get_jc_nt() + bli_thread_get_ic_nt() + bli_thread_get_jr_nt() + bli_thread_get_ir_nt() + bli_thread_get_num_threads() + bli_thread_set_jc_nt() + bli_thread_set_ic_nt() + bli_thread_set_jr_nt() + bli_thread_set_ir_nt() + bli_thread_set_num_threads() + - Added #include "errno.h" to bli_system.h. + - This commit addresses issue #140. + - Thanks to Chris Goodyer for inspiring these updates. + +commit 8772a0b33a90154c80d88b381dcdd66f824e041f +Author: Marat Dukhan +Date: Thu Jul 13 21:39:24 2017 -0700 + + Fix Emscripten builds + +commit 72c8b49bb8d3b9370b2cc37718da22f065de9c57 +Merge: 70cc825b ba7cada5 +Author: Field G. Van Zee +Date: Wed Jul 12 14:58:12 2017 -0500 + + Merge pull request #138 from hominhquan/membrk_set_free_fp + + Set missing free_fp in bli_membrk_init for free-ing GEN_USE buffers + +commit ba7cada51a238d320528e3504ed0f0a17a6b022a +Author: Minh Quan HO +Date: Fri Jul 7 10:52:05 2017 +0200 + + set missing free_fp in bli_membrk_init for free-ing GEN_USE buffers + + The membrk's free_fp is called when releasing GEN_USE buffers, but this free_fp is + not set in bli_membrk_init + +commit 1241301869957c96f16a2c6567e3ad70afa547de +Merge: 969b67e8 25ead66f +Author: Kiran Varaganti +Date: Wed Jul 5 02:24:00 2017 -0400 + + Merge "Reducing the framework overhead of GEMV routines" into amd-staging + +commit 25ead66fb78557f73af48bac305724d5d8aa3309 +Author: sthangar +Date: Fri Jun 30 12:23:19 2017 +0530 + + Reducing the framework overhead of GEMV routines + + Change-Id: I83607ad767bff74e305e915b54b0ea34ec3e5684 + +commit 969b67e8800fbd5d14a086606f3b5afbf66ed093 +Author: Kiran Varaganti +Date: Tue Jul 4 12:57:32 2017 +0530 + + Improved efficiency of dGEMM for large matrices by reducing TLB load misses and majorly L3 cache misses. This is achieved by changing the packed block sizes of matrix A & B. Now the optimum values are MC_D = 510 and KC_D = 1024. + + Change-Id: I2d8bdd5f62f2d1f8782ae2997f3d7a26587d1ca4 + +commit 70cc825b552dec05165b9d70f9e6eb33d8abb118 +Author: Devin Matthews +Date: Tue Jun 6 21:58:21 2017 -0500 + + Update LICENSE + + Remove totally unnecessary first 9 lines and hopefully get Github to recognize it as 3BSD [ci skip]. + +commit cf54c77bc79a0f33a514be72c80a654c4e6e6f63 +Author: Devin Matthews +Date: Tue Jun 6 20:23:17 2017 -0500 + + Add new SSI acknowledgment + +commit d6ef56c6dbaf6df8ee1af1ca6a0f0792a811396a +Author: prangana +Date: Thu Jun 1 16:11:09 2017 +0530 + + Update version number + + Change-Id: Ib6e52d1d34c0791367ab9152dfab31f94deedeb4 + +commit 897bfa0e92082c30bbb74229562d7d7327cbbac8 +Author: prangana +Date: Thu Jun 1 16:11:09 2017 +0530 + + Update version number + + Change-Id: Ib6e52d1d34c0791367ab9152dfab31f94deedeb4 + +commit 99d0ba5606d4b63e6a9c639aa78d4defc2455f79 +Merge: be2c7eb8 6d17e012 +Author: Santanu Thangaraj +Date: Thu Jun 1 02:19:02 2017 -0400 + + Merge "Checked in the small matrix code to compute GEMM called with A transpose case" into amd-staging + +commit 6d17e0120fe5c127b941136ad2c0c08e91439535 +Author: sthangar +Date: Wed May 24 11:48:16 2017 +0530 + + Checked in the small matrix code to compute GEMM called with A transpose case + + Change-Id: I29f40046d43d7a4b037c1cb322503ee26495f462 + +commit 9d93f8481a1404695f7b78a3ced8ca47e890b649 +Author: prangana +Date: Tue May 30 09:58:10 2017 +0530 + + Update Licence File + + Change-Id: I4c5cf1690d0cef92a68400f9a89e454ab6856ad2 + +commit be2c7eb85168937bd4318f4d05ded37620119310 +Author: prangana +Date: Tue May 30 09:58:10 2017 +0530 + + Update Licence File + + Change-Id: I4c5cf1690d0cef92a68400f9a89e454ab6856ad2 + +commit 7f41bb0a0becde6a7de7df0f99668d7b4686c3b0 +Author: Devin Matthews +Date: Fri May 26 14:49:31 2017 -0400 + + PACKDIM_MR=8 didn't work out, but messing with the prefetching helps 2%. + +commit d87614af3f3d9187be94d6e77984b282bf890928 +Author: Devin Matthews +Date: Fri May 26 14:47:36 2017 -0400 + + Revert "Change PACKDIM_MR (double) for haswell to 8." + + This reverts commit 681eec913d7c2ebcff637cec5c1627ced9a92b99. + +commit 681eec913d7c2ebcff637cec5c1627ced9a92b99 +Author: Devin Matthews +Date: Fri May 26 12:28:09 2017 -0500 + + Change PACKDIM_MR (double) for haswell to 8. + +commit 0a3ae0ecaa0ddcb5887005d7051fa234499f1120 +Merge: 0f4e6652 6e04f9df +Author: praveeng +Date: Sat May 20 16:53:50 2017 +0530 + + frame/3/gemm/bli_gemm_front.c + + Change-Id: I52a0fbc1d33bb948d430942323bbc5fe44e3ca13 + +commit 6e04f9df01d79c1b0e673943ca0d5d0a6095eb2e +Author: Field G. Van Zee +Date: Wed May 17 13:03:52 2017 -0500 + + Restored deleted lines from makefile fragments. + +commit ec5c0c0448275280dca0991f6f33afeb73650450 +Author: Devin Matthews +Date: Wed May 17 12:29:44 2017 -0500 + + Change to /bin/sh. + + All scripts checked with Debian's checkbashisms. Also check for clang first in auto-detect.sh. + +commit 555ddc30d4c7e44f3f335e436c98606f56e1598b +Author: Devin Matthews +Date: Wed May 17 12:27:14 2017 -0500 + + Remove shebangs from makefiles. + +commit f26bd7f42e0c2a47fe321b2c452644990b689654 +Merge: cbf8710a 169fb05f +Author: Devin Matthews +Date: Wed May 17 11:58:41 2017 -0500 + + Merge pull request #128 from iotamudelta/master + + Portability and clang + +commit 169fb05f225c2f060265bcaa872f7f80dc638b70 +Author: J M Dieterich +Date: Tue May 16 23:11:22 2017 -0400 + + Fix if/else structure. Thanks to TravisCI. + +commit 0579dfea0bcfbb90ebc073fcf78b92a5cf7238e1 +Author: J M Dieterich +Date: Tue May 16 22:58:07 2017 -0400 + + Restore version. + +commit a75b05c23dc786a1fdc45dc1627a5ce2299f1a7b +Author: J M Dieterich +Date: Tue May 16 22:23:27 2017 -0400 + + Mark piledriver compilable w/ clang. + +commit 7541d46e2ba8659bb2e36b444edef112fefa1345 +Author: J M Dieterich +Date: Tue May 16 22:12:12 2017 -0400 + + Mark bulldozer compilable w/ clang. + +commit 91f897073ec0df3330ede449c4d6af8158266ae3 +Author: J M Dieterich +Date: Tue May 16 22:06:59 2017 -0400 + + Correct error message. + +commit f5131e1e49167f948bddd714bb1af1761829c212 +Author: J M Dieterich +Date: Tue May 16 22:03:23 2017 -0400 + + Indeed once can compile for carrizo also using clang. + +commit 5fa4e9439c04f35f89dd7d26ff742cb2dadc3180 +Author: J M Dieterich +Date: Tue May 16 21:50:49 2017 -0400 + + A bunch of shebang fixes from unportable /bin/bash to portable /usr/bin/env bash + +commit 1f3a58197e5d5f9ac862bda91e7527cbfbab5d76 +Author: Field G. Van Zee +Date: Mon May 8 16:10:03 2017 -0500 + + Housekeeping, induced method file/function renames. + + Details: + - Renamed all level-3 induced method files to use the "_vir.c" suffix + instead of "_ref.c". Also renamed functions within these files + accordingly. + - Renamed cpp macro definitions in frame/ind/include according to the + above changes. + - Removed frame/3/old. + +commit cbf8710a1ba63e25aadaa6fc5da51ea81b3d596d +Merge: cf39d3ef fdc66f12 +Author: Tyler Michael Smith +Date: Mon May 8 11:21:20 2017 -0500 + + Merge pull request #127 from devinamatthews/fix_blis_nt_xx + + Setting any one of BLIS_NT_[IJ][CR] overrides BLIS_NUM_THEADS + +commit cf39d3ef3b29b8058c39fb4638c1a734fe64aaed +Author: Field G. Van Zee +Date: Fri May 5 15:06:56 2017 -0500 + + Fixed a bug in norm1v, norm1m. + + Details: + - Fixed a bug that manifested as improperly-computed 1-norm for vectors + and matrices. This is one of the few operations in BLIS that does not + have its own test module within the testsuite, hence why it went + undetected for so long. The bad 1-norms were being used to normalize + matrices in the testsuite after initialization, which led to some + matrices containing a combination of "large" and "small" values. This + tended to push the residuals computed after each test away from zero. + In some cases, they were off *just* enough to the testsuite to label + it a "failure". Many thanks to Jeff Hammond for reporting this bug. + (Wonky details: the bug was due to improperly-defined level-0 scalar + macros for abval2, an operation that computes the absolute square, + or complex magnitude/modulus. Certain complex domain instances of + abval2 were being incorrectly defined in terms of real-only solutions, + leading to bad results. This level-0 operation forms the basis of + norm1v/norm1m. absq2 was also affected, but almost nothing uses + this operation.) + +commit 799485124f4d823e908d2e5d38b0c3a1e6172ade +Merge: 773a24ef 0df3541f +Author: Devin Matthews +Date: Thu May 4 10:52:09 2017 -0500 + + Merge pull request #121 from jeffhammond/not-real-knl + + allow KNL build without hbwmalloc (i.e. emulated) + +commit fdc66f12d40754ff46179804bff592fddafbca02 +Author: Devin Matthews +Date: Thu May 4 10:35:22 2017 -0500 + + Setting any one of BLIS_NT_[IJ][CR] overrides BLIS_NUM_THEADS. Missing BLIS_NT_XX's are defaulted to 1. Fixes #123. + +commit 773a24efb2fa1c3a220bf0ce1dd621a3176196da +Merge: dd58c954 b8854259 +Author: Field G. Van Zee +Date: Wed May 3 15:07:59 2017 -0500 + + Merge branch 'master' of github.com:flame/blis + +commit dd58c9545c877c3f7553eaebca7b5e9720a66f5d +Author: Field G. Van Zee +Date: Wed May 3 15:04:51 2017 -0500 + + Disable complex 3m/4m in testsuite by default. + + Details: + - Disabled testsuite tests of all level-3 implementations based on 3m + and 4m. This will improve testing runtime on Travis CI as well as for + anyone manually running the testsuite using default test parameters. + Thanks to Devin Matthews for suggesting this change. + +commit 0df3541f54b7fe0c604ab2ec47ba814f12391798 +Author: Jeff Hammond +Date: Tue May 2 19:25:21 2017 -0700 + + allow KNL build without hbwmalloc.h (i.e. emulated) + + we want to be able to run BLIS KNL binaries on non-KNL machines via SDE. + although it is possible to install hbwmalloc implementation on such + systems, it is easier not to, since obviously the performance of SDE + execution is not representative so there is no reason to emulate HBW + allocation. + +commit b88542591d4dd0cde366e5ae35afd3205cb81bdc +Merge: 43007f7b c2c91e09 +Author: Field G. Van Zee +Date: Tue May 2 19:22:41 2017 -0500 + + Merge pull request #107 from jeffhammond/intel-compilers-no-use-libm + + never use libm with Intel compilers + +commit 43007f7b65ec7926cbbfc39965ff733fa251c15f +Author: Field G. Van Zee +Date: Tue May 2 16:48:43 2017 -0500 + + Fixed stray parentheses in README citations. + +commit a4f1d0b8801c114e9ef8be39df01e1b8d27ebcb3 +Author: Field G. Van Zee +Date: Tue May 2 16:38:43 2017 -0500 + + CHANGELOG update (0.2.2) + +commit 940a707ac78de975110e17c95765e65b89aa5e10 (tag: 0.2.2) Author: Field G. Van Zee Date: Tue May 2 16:38:42 2017 -0500 Version file update (0.2.2) -commit d5a5e003ea9b24bb6abf12e88862e8eb61ffb03d (origin/master, origin/HEAD, origin/1m, 1m) +commit d5a5e003ea9b24bb6abf12e88862e8eb61ffb03d Author: Field G. Van Zee Date: Tue May 2 15:48:30 2017 -0500 @@ -46,6 +2899,32 @@ Date: Tue May 2 12:09:39 2017 -0500 - Updated bibtex entries for 4th BLIS paper, and adds entries for 5th and 6th BLIS papers. +commit 0f4e6652dfe9b30105d3bab328ac26d9d5c11182 +Merge: 42e7f6fb 6e7de6ef +Author: praveeng +Date: Wed Apr 19 17:54:10 2017 +0530 + + Merge master code till 2017_04_19 to amd-staging + + Change-Id: Ibebe83c8ea2e7eb15798c2bcf214b7228a1c9518 + +commit 42e7f6fb2a531429ee600b2fe0293b67371c7ccb +Author: sthangar +Date: Tue Mar 28 18:10:03 2017 +0530 + + fixed license attribute issues in AMD added files + + Change-Id: I303f870a777c7cd1c1af29ea0b93f3e0a27948e4 + +commit 5600001e973c6cea048bd3fdb28117f1d7c98b9d +Merge: 0b190293 b3ed4933 +Author: prangana +Date: Mon Mar 20 13:56:33 2017 +0530 + + Fix merge conflicts after sync with release branch + + Change-Id: Icf14a09f728befb69a73fff9fa79c4128e728310 + commit 6e7de6ef84babb273dc5528a9b9d01f0febe394b Author: Field G. Van Zee Date: Fri Mar 17 12:10:24 2017 -0500 @@ -66,6 +2945,55 @@ Date: Fri Mar 17 12:07:27 2017 -0500 - Updated formatting of function signature/header in kernels/armv8a/3/bli_gemm_opt_4x4.c. +commit 0b19029342ffc530fa22ef20398a26221cb8f6ec +Author: Kiran Varaganti +Date: Tue Mar 14 14:51:31 2017 +0530 + + Code cleanup, removed warnings from trsm, removed unused routines in axpyv & scalv + + Change-Id: I02867f394c5f416194c4b1769a6c75f39243ec81 + +commit 825363bd2a5a60a923d4a6d9691dc143845a9cab +Merge: 093bdb80 513944e4 +Author: praveeng +Date: Wed Mar 8 15:42:49 2017 +0530 + + Merge code from master to amd-staging as on 2017_03_08 by praveeng + + Change-Id: I80740081b2cb54c9b77a3e78b9fe540e170be23d + +commit 093bdb80c86b06367e595aa17487139ae983822f +Author: sthangar +Date: Tue Mar 7 13:35:50 2017 +0530 + + Checked in Unpacked DGEMM code + + Change-Id: I39dcc7b238b328f73ee2675d21a5e521d0488723 + +commit 33923da9a108854590d386e74b6ee66b971e7796 +Author: Kiran Varaganti +Date: Mon Mar 6 14:31:31 2017 +0530 + + Added variant 10 for double precision axpyv microkernel + + Change-Id: I7a20cc113a422603250bc450825c965136354974 + +commit bc828f7f8e3ddb9f58af07edc0b935b21759fb0f +Author: Kiran Varaganti +Date: Fri Mar 3 14:45:35 2017 +0530 + + Added new axpyv (single precision) microkernel where it performs 10 FMAs per loop- This gives better performance than all other implementations of axpyv + + Change-Id: Ic4f0e4c67e367d67d0b24febcf34f81a70a39972 + +commit c9949f4603419267c10973adf1d63ec38497475d +Author: sthangar +Date: Fri Feb 17 14:16:33 2017 +0530 + + Checked in DGEMMTRSM and edge case handling routine in DDOTXF + + Change-Id: I65f00661af6c09b2507294fd43e0a10641c0597e + commit a509fbd5ac04fafd4e51b43d2f59ca56432dc212 Merge: 69b4846a 513944e4 Author: Field G. Van Zee @@ -116,6 +3044,14 @@ Date: Sun Feb 19 21:10:55 2017 -0500 Cast dim_t and inc_t parameters to 64-bit in KNL microkernels. +commit 04245c9ff7f8b3c70d61003029c964bb9a4320ee +Author: Kiran Varaganti +Date: Fri Feb 10 14:24:30 2017 +0530 + + Reoptimized scalv routines - two vector multiplies are done per iteration, and these routines are enabled in bli_kernel.h + + Change-Id: Ic5654508573d1f6bde2edef06aefe117e581feb5 + commit c362afc525bab4050581d1b0fcea2fe4d582c608 Author: Field G. Van Zee Date: Thu Feb 9 11:54:59 2017 -0600 @@ -139,6 +3075,60 @@ Date: Wed Feb 8 11:20:52 2017 -0600 which threading model was chosen. Thanks to heroxbd for reporting this issue. +commit 58b5b77e5fdb179ea465e398e416e6a00d917e05 +Author: Kiran Varaganti +Date: Wed Feb 8 21:43:34 2017 +0530 + + Fixed a bug in axpyv, the arguments passed to intrinsic fmad instruction are corrected + + Change-Id: If12f24c6bc74b22ac9e4acd6b9378e06d79f2f5e + +commit 85de4ebf74d0a5587d5a12724eb5489d51674db3 +Author: Kiran Varaganti +Date: Wed Feb 8 14:41:04 2017 +0530 + + variant 4 axpyv single precision modified: explicitly used FMA intrinsics, replaced vector multiply and add operations + + Change-Id: I975feef56696d479d2b9e9441b0660021cf4f6ff + +commit 3fa53e8af31d634779f40258c51483ae8af494fa +Merge: b5291a44 95be7b04 +Author: Kiran Varaganti +Date: Wed Feb 8 11:46:34 2017 +0530 + + Merged axpyv and gemm small in bli_kernel.h + Merge branch 'amd-staging' of ssh://git.amd.com:29418/cpulibraries/er/blis into amd-staging + + modified: config/zen/bli_kernel.h + modified: frame/3/gemm/bli_gemm_front.c + modified: kernels/x86_64/zen/3/bli_gemm_small_matrix.c + + Change-Id: If181cf9345178c448b3530beb8bef453917fe295 + +commit 95be7b04709e688a4cb01fba680081e30f4258ef +Author: sthangar +Date: Tue Feb 7 14:01:27 2017 +0530 + + Added logic for packing matrix A and prefetching matrix C in Unpacked SGEMM code + + Change-Id: I99efeca9eb5b4449286ec0ec133fd554ef1bb4f0 + +commit b5291a445b1313e01f1e0e8102c5f3660ab07f69 +Author: Kiran Varaganti +Date: Tue Feb 7 12:39:31 2017 +0530 + + Added optimization variant 4 for axpyv single precision - this performs 5 FMA per loop, keeping the IPC always full + + Change-Id: Ie77ed22584271136a257e673bcd3b1ba71136bc9 + +commit f4bfc1662af82aa4b98185334c44835e51f1cbec +Author: Kiran Varaganti +Date: Mon Feb 6 15:04:27 2017 +0530 + + New routines implemented for axpyv to improve performance for small vector sizes, vectorization is done for vectors as small as 8 (single precision) 4(double precision), since this operation has low compute to memory ratio, higher matrix sizes memory operations are dominating and hence not much gain - This still needs some work- added saxpyv and daxpyv var 3 routines in the file bli_axpyv_opt_var1.c + + Change-Id: Ic1b33bd5516e10113b00e44ab41b97eb19d46072 + commit ddf45e71770c55ea4a58ca24ea4913fe5d8beb9b Merge: a6ab91bc 78e1b16e Author: Devin Matthews @@ -154,6 +3144,14 @@ Date: Fri Jan 27 14:22:20 2017 -0600 Change default threading parameters for KNL. +commit 574472ba5a89924eca7dbd10055d0e1dcd7f4c71 +Author: sthangar +Date: Tue Jan 10 14:51:46 2017 +0530 + + checked in unpacked SGEMM optimization + + Change-Id: I8e4ea374415c0c402c660b656fb076af15354181 + commit 1c732d3ddc4ac0861d3b0e0dd15eb7e071615502 Author: Field G. Van Zee Date: Wed Jan 25 16:25:46 2017 -0600 @@ -206,6 +3204,23 @@ Date: Wed Jan 25 16:25:46 2017 -0600 cntx_t struct. - Comment updates. +commit 41595e98eedaf3f1f93802c14dcae490402f933f +Merge: d625c49e a6ab91bc +Author: praveeng +Date: Wed Dec 7 15:13:21 2016 +0530 + + Merge master code as on 2016_12_07 to amd-staging + + Change-Id: I5d9ecef9bff960aeb9b51ca4e4b21714e789e44f + +commit d625c49e20bd3c50d6d44e330e34076cced114a3 +Author: sthangar +Date: Tue Nov 29 15:05:19 2016 +0530 + + checked-in SGEMMTRSM microkernel for Zen + + Change-Id: Ib61936418dea911b2154aa99f703b66e9669f94f + commit a6ab91bc61432490fadf18d596de4589645f37dd Merge: 145a551d 7f31a630 Author: Field G. Van Zee @@ -264,6 +3279,22 @@ Date: Fri Nov 25 18:29:49 2016 -0600 - Added 1m support to Makefile and runme.sh in test/3m4m. Also simplified some code in test_gemm.c driver. +commit d8f13beeea90338e0ecb0a3aeaa2d59d8ebd6c36 +Merge: c25a9205 145a551d +Author: praveeng +Date: Fri Nov 25 17:31:08 2016 +0530 + + Merge master code till 2016_11_25 to amd-staging + +commit c25a9205fd8c8d8de7fd81b1e5621e7ac79f4e87 +Merge: 65298762 bdc0a264 +Author: praveeng +Date: Fri Nov 25 17:06:36 2016 +0530 + + Merge master code till Switched to simpler trsm_r 2016_11_25 to amd-staging + + Change-Id: Ibf71d224d8fb6cf0bc497f84d50c27d276512cc1 + commit 145a551d524ae5492667a05fc248923d922df850 Author: Field G. Van Zee Date: Wed Nov 23 17:59:06 2016 -0600 @@ -295,6 +3326,22 @@ Date: Wed Nov 23 17:58:26 2016 -0600 broadcast-based implementations. (The previous microkernel file has been moved to an 'old' subdirectory.) +commit 65298762ff15c45e8588e0c279a9feaa98c927a0 +Author: sthangar +Date: Tue Nov 22 12:15:33 2016 +0530 + + removed a redundant copy operation in DNRM2 + + Change-Id: I673b08efde4480e871779716f7715566740ad9ce + +commit d6863e851adeef037e4d1476fe63bb293fb9d987 +Author: sthangar +Date: Mon Nov 21 11:30:30 2016 +0530 + + checked-in DNRM2 optimizations + + Change-Id: I3b31d768bd7f4fbf43042aa5a0762995c73c4522 + commit bdc0a264d2fb5940bfd09298b1de823674a39053 Author: Field G. Van Zee Date: Wed Nov 16 14:13:08 2016 -0600 @@ -336,6 +3383,30 @@ Date: Wed Nov 16 14:04:33 2016 -0600 incorrectly at first. I've now fixed its location and changed its consequence to a compile-time #error message. +commit 9772218cae57d55c252595b01e3669d8bed84944 +Author: sthangar +Date: Wed Nov 16 15:19:19 2016 +0530 + + Added optimized DAMAX routines for Zen + + Change-Id: I499c0c8f0f4ce6c19235c47b86d5608db6ba50f8 + +commit 9c448e30174e5eb76a94b43b30819704a5dfcb3f +Merge: 998d8240 e35d3c23 +Author: Santanu Thangaraj +Date: Wed Nov 16 04:18:57 2016 -0500 + + Merge "Added new optimized micro-kernel for dotxv routine" into amd-staging + +commit 998d824044adac0d54c921dcd44fb58f3d54aad2 +Merge: 0d13e9a4 6b5a4032 +Author: praveeng +Date: Wed Nov 16 14:22:42 2016 +0530 + + Merge master code till devinamatthews/omp_num_thrds 2016_11_16 to amd-staging + + Change-Id: I601ff1d3ec8a680e1be039ffc7b299744e8a27c5 + commit 6b5a4032d2e3ed29a272c7f738b7e3ed6657e556 Merge: 3b524a08 a8220e3a Author: Field G. Van Zee @@ -352,6 +3423,23 @@ Date: Thu Nov 10 14:19:34 2016 -0600 - Fix typo in bli_cntx.c - Bump BLIS_DEFAULT_NR_THREAD_MAX to 4 +commit e35d3c23f28784e50ee13d2e77a69d60e0c24c1f +Author: Kiran Varaganti +Date: Thu Nov 10 14:30:53 2016 +0530 + + Added new optimized micro-kernel for dotxv routine + + Change-Id: I2c544e9b25a454d971ad690353502a55cd668391 + +commit 0d13e9a4f6f2fcda08f205215240cdf86442d6c6 +Merge: e044fa62 3b524a08 +Author: praveeng +Date: Mon Nov 7 14:40:41 2016 +0530 + + bli_kernel.h + + Change-Id: I425d089f79497a0de7d1622e829c3ca9edf7f091 + commit c05b3862f6241486442b313eff0c8bee7b5e1274 Author: Devin Matthews Date: Fri Nov 4 15:48:02 2016 -0500 @@ -485,6 +3573,20 @@ Date: Mon Oct 31 14:40:51 2016 -0500 subproblems (not ideal). This commit fixes this issue. Thanks to Devin for his suggestion. +commit c2c91e09b4893cb81314774557f728a95080f81e +Author: Jeff Hammond +Date: Tue Oct 25 21:15:26 2016 -0700 + + never use libm with Intel compilers + + Intel compilers include a highly optimized math library (libimf) that + should be used instead of GNU libm. + + yes, this change is for ALL targets, including those that are not + supported by the Intel compiler. there is no harm in doing this, and it + is future-proof in the event that the Intel compilers support other + architectures. + commit 630391002325a589063aec2ab0a7d89ef2e178c0 Merge: 956b3edf 216206c1 Author: Field G. Van Zee @@ -531,6 +3633,22 @@ Date: Tue Oct 25 12:42:44 2016 -0500 Add flexible options for thread model (pthread/posix for pthreads etc.). +commit e044fa624008c161de32a39d734cddf1dd22dd41 +Author: Kiran Varaganti +Date: Tue Oct 25 13:03:05 2016 +0530 + + Changed double precision trsm kernel macro definition to bli_dtrsm_l_int_6x8 from 6x16 : it fixes the seg fault + + Change-Id: Ia8c1de5fe13a370d691570a50136d55ffb18908a + +commit b3ed4933aa0da72ad771fb0fdf1727e5ba9ad7b4 +Author: Kiran Varaganti +Date: Tue Oct 25 13:03:05 2016 +0530 + + Changed double precision trsm kernel macro definition to bli_dtrsm_l_int_6x8 from 6x16 : it fixes the seg fault + + Change-Id: Ia8c1de5fe13a370d691570a50136d55ffb18908a + commit b7e41d71b07d2af6d22d632c70e0c5f7ce46852c Merge: 4bd905bd 5117d444 Author: Field G. Van Zee @@ -569,6 +3687,14 @@ Date: Fri Oct 21 14:34:27 2016 -0500 sanity check that OpenMP and POSIX threads are not both enabled. - Thanks to Krzysztof Drewniak for reporting this bug. +commit d250e6a3af3af8beedcda28f508ac03e94efb3c8 +Author: Kiran Varaganti +Date: Thu Oct 20 14:34:39 2016 +0530 + + Merged TRSM and scalv routines into zen folder + + Change-Id: Ice897bc83e8fb70b90f23cc3ce892c39883aceb9 + commit 8feb0f85a674e84bec2417486e3bcea584b14c04 Author: Field G. Van Zee Date: Wed Oct 19 16:05:41 2016 -0500 @@ -612,6 +3738,23 @@ Date: Wed Oct 19 15:58:03 2016 -0500 - The redundant typedefs of membrk_t and mtx_t caused a warning on some C compilers. Thanks to Tyler Smith for reporting this issue. +commit 1c2f7b57d557c05f5ef6148cccafaf0f70d910da +Author: sthangar +Date: Tue Oct 18 15:06:35 2016 +0530 + + Removed symlinks to zen kernels from haswell kernel folder and also modified the bli_kernel.h file accordingly + + Change-Id: Ib3736af48e851c8243bbe10d937fb942c49ad048 + +commit d864ea9f4f039fe2b2dc395d0015bd9e8902bc8e +Merge: 7045fcbf 28b2af8a +Author: praveeng +Date: Fri Oct 14 17:00:57 2016 +0530 + + Merge master code 2016_10_14 till Added disabled code thrinfo_t structures + + Change-Id: If7db98d286c1471fcd30f00757abee9b253ef987 + commit 28b2af8a71133ce68774e153b6e05afb05affba8 Author: Field G. Van Zee Date: Thu Oct 13 14:50:08 2016 -0500 @@ -639,6 +3782,23 @@ Date: Thu Oct 13 14:23:23 2016 -0500 commit fd04869, which changed the preferred configure option string for enabling OpenMP from "omp" to "openmp". +commit 7045fcbf0bd349ebe6cb9ac4508c6a387bb05966 +Merge: 7e044900 9cda6057 +Author: praveeng +Date: Thu Oct 13 12:02:28 2016 +0530 + + Merge master code 2016_10_13 Removed previously renamed/old files + + Change-Id: I8106d371afaa0af474a8967388d44481b05de923 + +commit 7e04490002206d3557fcfb7dd893838a7f36916f +Author: sthangar +Date: Wed Oct 12 16:43:02 2016 +0530 + + Checked in the SAMAX optimizations + + Change-Id: I7faf8c3adf52ff01432188ad3b9866ee4b9a9dfd + commit 9cda6057eaa16a24ac8785a9fa167df6c9edba44 Author: Field G. Van Zee Date: Tue Oct 11 13:21:26 2016 -0500 @@ -680,6 +3840,31 @@ Date: Thu Oct 6 14:48:15 2016 -0500 order to free the address immediately before the pointer. Thanks to Devin Matthews for reporting this bug. +commit cd84fb95182514601d72c78ee0e36a394d0284d7 +Author: praveeng +Date: Thu Oct 6 15:08:21 2016 +0530 + + syntax erros in configure file + + Change-Id: Ibe8a6071aad97df550df64c009fec33a9d8f43a1 + +commit f2e7ea113aa93b74f1d42408d5db2c5a7b00a653 +Merge: 133983c3 86969873 +Author: praveeng +Date: Thu Oct 6 12:35:30 2016 +0530 + + conflicts merge for bli_kernel.h + + Change-Id: I15d846bd34e11f86ebfd7ed091ff671a1f3366a0 + +commit 133983c36fa01c7acb6d666b3744f77f216314a5 +Author: sthangar +Date: Thu Oct 6 11:26:22 2016 +0530 + + code clean up in bli_kernel.h + + Change-Id: I11d9cdf2af8e8199209eb084f6c3a7c910b83d5d + commit 4fb9b4ef2e4cf2626a6e000a41628fb823f16da8 Author: Field G. Van Zee Date: Wed Oct 5 14:41:35 2016 -0500 @@ -699,7 +3884,7 @@ Date: Wed Oct 5 13:35:01 2016 -0500 Merge branch 'compose' -commit 6f71cd344951854e4cff9ea21bbdfe536e72611d (origin/compose, compose) +commit 6f71cd344951854e4cff9ea21bbdfe536e72611d (origin/compose) Merge: c0630c40 8d55033c Author: Field G. Van Zee Date: Tue Oct 4 15:53:46 2016 -0500 @@ -735,7 +3920,7 @@ Date: Tue Oct 4 14:24:59 2016 -0500 of the complex domain). - Removed the directory frame/include/old/. -commit 8d55033c966feed99fcca2a58017c3ab5b1646dc (origin/distcomm) +commit 8d55033c966feed99fcca2a58017c3ab5b1646dc Author: Field G. Van Zee Date: Tue Sep 27 15:20:58 2016 -0500 @@ -807,6 +3992,47 @@ Date: Fri Sep 16 09:29:28 2016 -0500 Fixes broken URL in README.md +commit b922d7563422e14c49a4677bc6ae088a408861ed +Author: Field G. Van Zee +Date: Tue Aug 23 13:38:36 2016 -0500 + + Avoid compiling BLAS/CBLAS files when disabled. + + Details: + - Updated the top-level Makefile, build/config.mk.in template, and + configure script so that object files corresponding to source files + belonging to the BLAS compatibility layer are not compiled (or archived) + when the compatibility layer is disabled. (Same for CBLAS.) Thanks + to Devin Matthews for suggesting this optimization. + - Slight change to the way configure handles internal variables. Instead + of converting (overwriting) some, such as enable_blas2blis and + enable_cblas, from a "yes" or "no" to a "1" or "0" value, the latter are + now stored in new variables that live alongside the originals (with the + suffix "_01"). This is convenient since some values need to be + sed-substituted into the config.mk.in template, which requires "yes" or + "no", while some need to be written to the bli_config.h.in template, + which requires "0" or "1". + + Updated BLIS4 TOMS citation in README.md. + + Added complex gemm micro-kernels for haswell. + + Details: + - Defined cgemm (3x8) and zgemm (3x4) micro-kernels for haswell-based + architectures. As with their real domain brethren, these kernels perfer + row storage, (though this doesn't affect most users due to high-level + optimizations in most level-3 operations that induce a transpose to + whatever storage preference the kernel may have). + + Change-Id: I512ab90784ecbb7cdaee24928d2ccebb544ba5c1 + +commit 69826110bab2a064ec76457c24843d28f2581281 +Merge: 64598ee4 a58dd35e +Author: Pradeep Rao +Date: Wed Sep 14 03:26:25 2016 -0400 + + Merge "Implemented trsm single precision for lower triangular matrices, files added bli_trsm_l_int_6x16.cfiles modified bli_kernel.h to enable optimized trsm microkernel and test_trsm.c is modified to test trsm single precision" into amd-staging + commit c0630c4024b08750043a2942a3e8a037aa6b6259 Author: Field G. Van Zee Date: Mon Sep 12 13:59:02 2016 -0500 @@ -854,12 +4080,36 @@ Date: Wed Aug 31 17:34:15 2016 -0500 in bli_gemm3m3_packa() on the bli_obj_scalar_reset() on C. - Thanks to Tyler Smith for help with these changes. +commit 64598ee4cfb86f64abbd4bcef5a82ba0d5565b67 +Author: sthangar +Date: Wed Aug 31 12:54:50 2016 +0530 + + fixed the symlink issue + + Change-Id: I2186d529f295c576597c189e1ae219bc1a83f955 + commit abd61f9fa75d77a96d1491b3e035451ee73238fe Author: Field G. Van Zee Date: Tue Aug 30 12:34:19 2016 -0500 Updated BLIS4 TOMS citation in README.md. +commit 8a2373f26ba8fcd5b2d7b2cc72cb8b2e1f841a03 +Author: sthangar +Date: Mon Aug 29 14:10:45 2016 +0530 + + Norm 2 optimization + + Change-Id: Ide9decaccd20bf0ccc32c9abb6556e038dceed2b + +commit fdc663902347aa252ea88cf09ce24ab748958dff +Author: sthangar +Date: Mon Aug 29 10:43:38 2016 +0530 + + Placed 1 and 1f AMD optimized AVX routines under zen folder + + Change-Id: I26795211ef11d232ed794ce36dd0a9c1f8706328 + commit 701b9aa3ff028decbf90efac0dca5bd64fe26269 Author: Field G. Van Zee Date: Fri Aug 26 19:04:45 2016 -0500 @@ -980,6 +4230,14 @@ Date: Fri Aug 26 19:04:45 2016 -0500 reference micro-kernels for complex datatypes, and testing these implementations can slow down the testsuite considerably. +commit a58dd35ed7b5b77a6b272655d2edd7a822b8fa87 +Author: Kiran Varaganti +Date: Fri Aug 26 14:55:12 2016 +0530 + + Implemented trsm single precision for lower triangular matrices, files added bli_trsm_l_int_6x16.cfiles modified bli_kernel.h to enable optimized trsm microkernel and test_trsm.c is modified to test trsm single precision + + Change-Id: Ibddf989f4aad577e89558673e1038cf6ece654d9 + commit 73517f522b69de429dd7f3df60a70c068149ab28 Merge: c6f5c215 50293da3 Author: Field G. Van Zee @@ -1008,6 +4266,19 @@ Date: Tue Aug 23 13:38:36 2016 -0500 "no", while some need to be written to the bli_config.h.in template, which requires "0" or "1". +commit 22dd6a353ddb56614309c01533b1a94c9fd32bca +Merge: cdfb3c3f f20ed388 +Author: praveeng +Date: Tue Aug 23 15:15:35 2016 +0530 + + Merge master code as on 2016_08_23 to amd-staging branch by praveeng + + Changes to be committed: + modified: frame/thread/bli_mutex_openmp.h + modified: frame/thread/bli_mutex_pthreads.h + + Change-Id: Ica522edbb1d0173f53f38d5057b1f7aef73666be + commit c6f5c215ee793d03ea834469fc2adc53feaffc42 Merge: d52cb767 16a4c7a8 Author: Field G. Van Zee @@ -1015,6 +4286,125 @@ Date: Mon Aug 22 17:33:02 2016 -0500 Merge branch 'master' into compose +commit f20ed3885d628992fab88690f629a5a2bab3eb88 +Merge: 02ac597e 4bc842ca +Author: praveeng +Date: Mon Aug 22 15:27:33 2016 +0530 + + Merge branch 'master' of https://github.com/clMathLibraries/blis-amd for "Fixed bugs in bli_mutex_init() and friends." + +commit 02ac597e4b9be2670d9fff65d28552f8e1ec81b3 +Author: praveeng +Date: Thu Jul 28 15:11:08 2016 +0530 + + Revert commits 357c990bdd7bd5667aac5adf1bab3712973e7414 + + Change-Id: I12a34456d7eed93fda4369e76bcddb42ba7ccb99 + +commit 84e41cc73c9c87ce64582acd4264b8e1b5316482 +Author: praveeng +Date: Thu Jul 28 15:01:36 2016 +0530 + + Revert commits 8aee306 + + Change-Id: I3dd999c77c6779332a40dbb84371ca487216f189 + +commit 30ccfcee82db93d0109d1571242e2db925e95d0a +Author: praveeng +Date: Mon Jul 25 14:14:00 2016 +0530 + + removed changes from readme file which are giving confilcts + + Change-Id: Ic71ad1313e1404fed444e899466043704d875af6 + +commit aeca25cd63fc8971f8fe7809599c57853f976548 +Author: praveeng +Date: Tue Jul 5 16:51:23 2016 +0530 + + first commit + + Change-Id: Ib50c81acda3b2c1583da3d421efc0ca547ef68e2 + +commit 6b2274864b36fd1019d97bcc4ca6dd7a57ef16d9 +Author: praveeng +Date: Tue Jul 5 15:00:31 2016 +0530 + + small modification to readme for git push test + + Change-Id: I68506a49586b07eaa907f3f85304ee40d4c92d0a + +commit daa7a9ecb25982f2551adbd95e65f8ba97cfe944 +Author: praveeng +Date: Tue Jul 5 16:51:23 2016 +0530 + + first commit + + Change-Id: Ib50c81acda3b2c1583da3d421efc0ca547ef68e2 + +commit 5f66a4aa05aeffcb6eb587851d78d9527319466c +Author: praveeng +Date: Tue Jul 5 15:00:31 2016 +0530 + + small modification to readme for git push test + + Change-Id: I68506a49586b07eaa907f3f85304ee40d4c92d0a + +commit c6cbd78d2388c08824822b91a1c36ac4349bb67f +Author: praveeng +Date: Thu Jul 28 15:11:08 2016 +0530 + + Revert commits 357c990bdd7bd5667aac5adf1bab3712973e7414 + + Change-Id: I12a34456d7eed93fda4369e76bcddb42ba7ccb99 + +commit 9219a9060762525f87ebbf556d78fe8621858513 +Author: praveeng +Date: Thu Jul 28 15:01:36 2016 +0530 + + Revert commits 8aee306 + + Change-Id: I3dd999c77c6779332a40dbb84371ca487216f189 + +commit 728573296efa7cf14d2381570e116509dfe2a240 +Author: praveeng +Date: Mon Jul 25 14:14:00 2016 +0530 + + removed changes from readme file which are giving confilcts + + Change-Id: Ic71ad1313e1404fed444e899466043704d875af6 + +commit ad7862e291c240505c733a41d231b1a126ade73c +Author: praveeng +Date: Tue Jul 5 16:51:23 2016 +0530 + + first commit + + Change-Id: Ib50c81acda3b2c1583da3d421efc0ca547ef68e2 + +commit ad4b471a25ce77867295e5529dfc787e7c18b03f +Author: praveeng +Date: Tue Jul 5 15:00:31 2016 +0530 + + small modification to readme for git push test + + Change-Id: I68506a49586b07eaa907f3f85304ee40d4c92d0a + +commit 55d641363fcd8bdfdabbd7c22822fa2d0b7f3fa6 +Author: praveeng +Date: Tue Jul 5 16:51:23 2016 +0530 + + first commit + + Change-Id: Ib50c81acda3b2c1583da3d421efc0ca547ef68e2 + +commit f3b6b15f6d591d323802bd6c81c522a02056506d +Author: praveeng +Date: Tue Jul 5 15:00:31 2016 +0530 + + small modification to readme for git push test + + Change-Id: I68506a49586b07eaa907f3f85304ee40d4c92d0a + commit 16a4c7a823d60707ed9272f5d36e5c5d54c0ba4b Author: Field G. Van Zee Date: Fri Aug 19 11:38:36 2016 -0500 @@ -1061,6 +4451,94 @@ Date: Wed Aug 3 11:28:24 2016 -0500 This version gets ~1550 GFLOPs on KNL wuth 16x4. +commit cdfb3c3f29d321033fca106aa58ab67ead90a95d +Merge: 50a2f2ef 4bc842ca +Author: praveeng +Date: Fri Jul 29 12:45:04 2016 +0530 + + Merge master code as on 2016_07_29 to amd-staging branch by praveeng + + Change-Id: Ic78b84d8b8d10158fb2a612f9a64bbc7b1f9b486 + +commit 4bc842ca3a64e658c0808bfe4c5693a5ace97923 +Merge: 117f8838 b0d510bf +Author: praveeng +Date: Thu Jul 28 17:32:12 2016 +0530 + + Merge branch 'master' of publicrepo + +commit 117f8838511a478aa16137e770d27dd21f4227c5 +Author: praveeng +Date: Thu Jul 28 15:11:08 2016 +0530 + + Revert commits 357c990bdd7bd5667aac5adf1bab3712973e7414 + + Change-Id: I12a34456d7eed93fda4369e76bcddb42ba7ccb99 + +commit 2fcdc28f1055d385b2e662aa920fb97c472394d7 +Author: praveeng +Date: Thu Jul 28 15:01:36 2016 +0530 + + Revert commits 8aee306 + + Change-Id: I3dd999c77c6779332a40dbb84371ca487216f189 + +commit 1b5d104afe0628b8b6c0650f1e58cfb08be67004 +Author: praveeng +Date: Mon Jul 25 14:14:00 2016 +0530 + + removed changes from readme file which are giving confilcts + + Change-Id: Ic71ad1313e1404fed444e899466043704d875af6 + +commit d81273047bff56501e9413a90991d3d1f8b56a06 +Author: praveeng +Date: Tue Jul 5 16:51:23 2016 +0530 + + first commit + + Change-Id: Ib50c81acda3b2c1583da3d421efc0ca547ef68e2 + +commit 65905c3011a11cda95761681d4ae84337e46bdb5 +Author: praveeng +Date: Tue Jul 5 15:00:31 2016 +0530 + + small modification to readme for git push test + + Change-Id: I68506a49586b07eaa907f3f85304ee40d4c92d0a + +commit 23cca231be10fe1797aed451bcbc69d38c78bc0c +Author: praveeng +Date: Tue Jul 5 16:51:23 2016 +0530 + + first commit + + Change-Id: Ib50c81acda3b2c1583da3d421efc0ca547ef68e2 + +commit 922e3091702f25e3287b417719a33adbd5bbf138 +Author: praveeng +Date: Tue Jul 5 15:00:31 2016 +0530 + + small modification to readme for git push test + + Change-Id: I68506a49586b07eaa907f3f85304ee40d4c92d0a + +commit b0d510bf0e4dfd177f9e4ae0069f41921e2ecdc1 +Author: praveeng +Date: Thu Jul 28 15:11:08 2016 +0530 + + Revert commits 357c990bdd7bd5667aac5adf1bab3712973e7414 + + Change-Id: I12a34456d7eed93fda4369e76bcddb42ba7ccb99 + +commit 5ebeece5b4a8df81d59ca7558b278a4263d15128 +Author: praveeng +Date: Thu Jul 28 15:01:36 2016 +0530 + + Revert commits 8aee306 + + Change-Id: I3dd999c77c6779332a40dbb84371ca487216f189 + commit 6ce4c022ebdea00c2b951090e3c2e9e88735b9ce Author: Devin Matthews Date: Wed Jul 27 16:26:36 2016 -0500 @@ -1158,6 +4636,30 @@ Date: Mon Jul 25 10:02:25 2016 -0500 Minor fixes for 8x24 KNL kernel. +commit 50a2f2efcbeb46537f1deaa8e44dc579a4e49eb8 +Merge: 1aa77dfc cfd46c88 +Author: praveeng +Date: Mon Jul 25 17:01:20 2016 +0530 + + Merge master code as on 2016_07_25 to amd-staging branch by praveeng + + Change-Id: I84886ae241db2aac0bef6b7ef399f04aa8bca16d + +commit cfd46c88d59c8f61d5e7cf768d606e4c44623584 +Merge: f493bf4d a017062f +Author: praveeng +Date: Mon Jul 25 15:38:13 2016 +0530 + + Merge remote-tracking branch 'publicrepo/master' + +commit f493bf4d704fe0e967783cd6e6877d3302c056a1 +Author: praveeng +Date: Mon Jul 25 14:14:00 2016 +0530 + + removed changes from readme file which are giving confilcts + + Change-Id: Ic71ad1313e1404fed444e899466043704d875af6 + commit 65735bbedf75784c48bd11e05b3fdc98fc66b4bc Author: Devin Matthews Date: Sun Jul 24 21:50:32 2016 -0500 @@ -1310,6 +4812,15 @@ Date: Fri Jul 22 10:23:31 2016 -0500 Add 8x24 KNL kernel. +commit 1aa77dfc1dc183d16e0b6a1196d9c263f021e83d +Merge: 9101a9c8 ec9f5983 +Author: praveeng +Date: Thu Jul 21 14:22:40 2016 +0530 + + Merge master code as on 2016_07_21 to amd-staging branch by praveeng + + Change-Id: Ic7d0a21101358f08147736e7f1884e7409937344 + commit b58cda9eba0c1e175460aae109baf792d29ba5bf Merge: 318f063d 413d62ac Author: Devin Matthews @@ -1322,6 +4833,29 @@ Date: Tue Jul 19 14:09:09 2016 -0500 # frame/include/blis.h # frame/thread/bli_thread.c +commit ec9f59836b32260c29ff1cd24e629c7d8de14992 +Merge: 197e182f 763babe4 +Author: praveeng +Date: Mon Jul 18 12:56:25 2016 +0530 + + Merge branch 'master' of https://github.com/clMathLibraries/blis-amd + +commit 197e182fcbf1340fd4a202fac58bea6cfcfa9e2f +Author: praveeng +Date: Tue Jul 5 16:51:23 2016 +0530 + + first commit + + Change-Id: Ib50c81acda3b2c1583da3d421efc0ca547ef68e2 + +commit 41fb32711031e7ec86b062aa7f53255d1f5905e2 +Author: praveeng +Date: Tue Jul 5 15:00:31 2016 +0530 + + small modification to readme for git push test + + Change-Id: I68506a49586b07eaa907f3f85304ee40d4c92d0a + commit d0dfe5b5372cc7558ee9c4104b29f82eecc7ed61 Merge: 31def12e 413d62ac Author: Field G. Van Zee @@ -1329,6 +4863,21 @@ Date: Thu Jul 14 11:01:06 2016 -0500 Merge branch 'master' into compose +commit 9101a9c880e3934f8a63ffc7fe15f5fc1077a73d +Author: sthangar +Date: Wed Jul 13 16:51:14 2016 +0530 + + Checked in optimized 1V kernels along with benchmark codes. Also incorporated review comments for 1F kernels + + Change-Id: I035c0d39e6b0bed28e6e2041242186c49f6ed55b + +commit 763babe488880b42c86c7fc207aa7665bd0ff9f7 +Merge: 357c990b 413d62ac +Author: praveeng +Date: Wed Jul 13 11:57:19 2016 +0530 + + Merge remote-tracking branch 'publirepo/master' + commit 413d62aca28edabba56605a9f87d5b715831e1db Author: Field G. Van Zee Date: Tue Jul 12 15:02:52 2016 -0500 @@ -1341,6 +4890,22 @@ Date: Tue Jul 12 14:21:19 2016 -0500 README update (BLIS2 TOMS article now in-print). +commit 357c990bdd7bd5667aac5adf1bab3712973e7414 +Author: praveeng +Date: Tue Jul 5 16:51:23 2016 +0530 + + first commit + + Change-Id: Ib50c81acda3b2c1583da3d421efc0ca547ef68e2 + +commit 8aee306300adb099b66036f2c2f7f3996433cf49 +Author: praveeng +Date: Tue Jul 5 15:00:31 2016 +0530 + + small modification to readme for git push test + + Change-Id: I68506a49586b07eaa907f3f85304ee40d4c92d0a + commit 31def12e2629f187e40f93f6bae9e26a6c2660e2 Author: Field G. Van Zee Date: Thu Jun 30 15:19:20 2016 -0500 @@ -1372,6 +4937,14 @@ Date: Thu Jun 30 15:19:20 2016 -0500 - Function signature (whitespace) reformatting for various functions. - Removed old code in various 'old' directories. +commit 405c9d46344d93c3eab5572b233900b50ca50d68 +Author: sthangar +Date: Wed Jun 22 12:18:54 2016 +0530 + + Check-in the fused kernels optimized for Zen + + Change-Id: I7b2f467b960e7b9a285f06e47be87de122e5fa24 + commit 232754feecf29452987666b9f5ebba2619bfd0b0 Author: Field G. Van Zee Date: Tue Jun 21 14:25:39 2016 -0500 From d12d34e167d7dc32732c0ed135f8065a55088106 Mon Sep 17 00:00:00 2001 From: Nisanth M P Date: Mon, 19 Mar 2018 11:34:32 +0530 Subject: [PATCH 02/53] Re-enabling Zen optimized cache block sizes for config target zen Change-Id: I8191421b876755b31590323c66156d4a814575f1 --- config/zen/bli_cntx_init_zen.c | 8 ++++++-- config/zen/bli_family_zen.h | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/config/zen/bli_cntx_init_zen.c b/config/zen/bli_cntx_init_zen.c index d3c81f709..6272f074e 100644 --- a/config/zen/bli_cntx_init_zen.c +++ b/config/zen/bli_cntx_init_zen.c @@ -110,10 +110,14 @@ void bli_cntx_init_zen( cntx_t* cntx ) // s d c z bli_blksz_init_easy( &blkszs[ BLIS_MR ], 6, 6, 3, 3 ); bli_blksz_init_easy( &blkszs[ BLIS_NR ], 16, 8, 8, 4 ); - //bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 510, 144, 72 ); - //bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 1024, 256, 256 ); +#ifdef BLIS_ENABLE_ZEN_BLOCK_SIZES + // Zen optmized level 3 cache block sizes + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 510, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 1024, 256, 256 ); +#else bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); +#endif bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); bli_blksz_init_easy( &blkszs[ BLIS_AF ], 8, 8, 8, 8 ); bli_blksz_init_easy( &blkszs[ BLIS_DF ], 8, 8, 8, 8 ); diff --git a/config/zen/bli_family_zen.h b/config/zen/bli_family_zen.h index 90aed8810..ccf6a2d42 100644 --- a/config/zen/bli_family_zen.h +++ b/config/zen/bli_family_zen.h @@ -42,7 +42,7 @@ #define BLIS_DEFAULT_MR_THREAD_MAX 1 #define BLIS_DEFAULT_NR_THREAD_MAX 1 - +#define BLIS_ENABLE_ZEN_BLOCK_SIZES //#define BLIS_ENABLE_SMALL_MATRIX // This will select the threshold below which small matrix code will be called. From fc53ad6c5b2e39238b1bbbf625cc0c638b9da4e1 Mon Sep 17 00:00:00 2001 From: Nisanth M P Date: Mon, 19 Mar 2018 12:49:26 +0530 Subject: [PATCH 03/53] Re-enabling the small matrix gemm optimization for target zen Change-Id: I13872784586984634d728cd99a00f71c3f904395 --- config/zen/bli_family_zen.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/zen/bli_family_zen.h b/config/zen/bli_family_zen.h index ccf6a2d42..db9fd4e8f 100644 --- a/config/zen/bli_family_zen.h +++ b/config/zen/bli_family_zen.h @@ -43,12 +43,12 @@ #define BLIS_DEFAULT_NR_THREAD_MAX 1 #define BLIS_ENABLE_ZEN_BLOCK_SIZES -//#define BLIS_ENABLE_SMALL_MATRIX +#define BLIS_ENABLE_SMALL_MATRIX // This will select the threshold below which small matrix code will be called. -//#define BLIS_SMALL_MATRIX_THRES 700 -//#define BLIS_SMALL_M_RECT_MATRIX_THRES 160 -//#define BLIS_SMALL_K_RECT_MATRIX_THRES 128 +#define BLIS_SMALL_MATRIX_THRES 700 +#define BLIS_SMALL_M_RECT_MATRIX_THRES 160 +#define BLIS_SMALL_K_RECT_MATRIX_THRES 128 From 3f1ba4e646776699ebfaa042fe24691d9e2f55d0 Mon Sep 17 00:00:00 2001 From: sraut Date: Tue, 5 Jun 2018 14:21:13 +0530 Subject: [PATCH 04/53] copyright changed to 2018 Change-Id: Ie916c7cd6f95aedc3cab6eec3a703c9ddb333bc3 --- config/zen/bli_family_zen.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/zen/bli_family_zen.h b/config/zen/bli_family_zen.h index db9fd4e8f..9b34213b0 100644 --- a/config/zen/bli_family_zen.h +++ b/config/zen/bli_family_zen.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2016, Advanced Micro Devices, Inc + Copyright (C) 2018, Advanced Micro Devices, Inc Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are From d4c24ea5f644eb635046e7fe249d3e8e58b4c98a Mon Sep 17 00:00:00 2001 From: sraut Date: Tue, 5 Jun 2018 15:42:59 +0530 Subject: [PATCH 05/53] copyright message changed to 2018 Change-Id: I33c1ebda41bc7f1973ff19e3b1947bdad62b4d44 --- kernels/zen/3/bli_gemm_small.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernels/zen/3/bli_gemm_small.c b/kernels/zen/3/bli_gemm_small.c index 47e850026..81c2525e8 100644 --- a/kernels/zen/3/bli_gemm_small.c +++ b/kernels/zen/3/bli_gemm_small.c @@ -4,7 +4,7 @@ BLIS An object-based framework for developing high-performance BLAS-like libraries. -Copyright (C) 2017, Advanced Micro Devices, Inc. +Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are From df1dd24fd896821de60917b429f303bab7fd0d4b Mon Sep 17 00:00:00 2001 From: sraut Date: Wed, 6 Jun 2018 11:24:33 +0530 Subject: [PATCH 06/53] small matrix trsm intrinsics optimization code for AX=B and XA'=B Change-Id: I90123c4d9adbd314c867995cd19dc975150b448c --- config/zen/bli_family_zen.h | 4 +- frame/3/trsm/bli_trsm_front.c | 49 +- frame/3/trsm/bli_trsm_front.h | 11 + kernels/zen/3/bli_trsm_small.c | 5487 ++++++++++++++++++++++++++++++++ 4 files changed, 5548 insertions(+), 3 deletions(-) create mode 100644 kernels/zen/3/bli_trsm_small.c diff --git a/config/zen/bli_family_zen.h b/config/zen/bli_family_zen.h index 9b34213b0..f61ec96d0 100644 --- a/config/zen/bli_family_zen.h +++ b/config/zen/bli_family_zen.h @@ -44,13 +44,15 @@ #define BLIS_ENABLE_ZEN_BLOCK_SIZES #define BLIS_ENABLE_SMALL_MATRIX +#define BLIS_ENABLE_SMALL_MATRIX_TRSM // This will select the threshold below which small matrix code will be called. #define BLIS_SMALL_MATRIX_THRES 700 #define BLIS_SMALL_M_RECT_MATRIX_THRES 160 #define BLIS_SMALL_K_RECT_MATRIX_THRES 128 - +#define BLIS_SMALL_MATRIX_THRES_TRSM 32768 //128(128+128) => m*(m+n) +#define BLIS_SMALL_MATRIX_A_THRES_TRSM 128 //#endif diff --git a/frame/3/trsm/bli_trsm_front.c b/frame/3/trsm/bli_trsm_front.c index 10817bee1..a072d5cb7 100644 --- a/frame/3/trsm/bli_trsm_front.c +++ b/frame/3/trsm/bli_trsm_front.c @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -33,7 +34,7 @@ */ #include "blis.h" - +//#define PRINT_SMALL_TRSM_INFO void bli_trsm_front ( side_t side, @@ -45,10 +46,54 @@ void bli_trsm_front ) { bli_init_once(); - + int i, j; obj_t a_local; obj_t b_local; obj_t c_local; +int m = bli_obj_length(*b); +int n = bli_obj_width(*b); +float *L = a->buffer; + float *B = b->buffer; + +#ifdef PRINT_SMALL_TRSM_INFO + printf("Side:: %c\n", side ? 'R' : 'L'); + if (bli_obj_datatype(*a) == BLIS_FLOAT) + printf("Alpha:: %9.2e\n", *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, *alpha))); + else if (bli_obj_datatype(*a) == BLIS_DOUBLE) + printf("Alpha is double:: %9.2e\n", *((double *)bli_obj_buffer_for_const(BLIS_DOUBLE, *alpha))); + else + printf("Unsupported datatype for Alpha\n"); + + printf("A:: M = %d, N = %d, elem_size = %d, row_off = %ld, col_off = %ld, rs = %d, cs = %d, trans = %c, TRIANG = %c, unit diag = %c\n", a->dim[0], a->dim[1], bli_obj_elem_size(*a ), bli_obj_row_off(*a), bli_obj_col_off(*a), a->rs, a->cs, bli_obj_has_trans(*a) ? 'Y' : 'N', bli_obj_is_upper(*a) ? 'U' : bli_obj_is_lower(*a) ? 'L' : 'N', bli_obj_has_unit_diag(*a) ? 'Y' : 'N'); +#ifdef PRINT_SMALL_TRSM + //bli_printm("a", a, "%4.1f", ""); +#endif + printf("B:: M = %d, N = %d, elem_size = %d, row_off = %ld, col_off = %ld, rs = %d, cs = %d, trans = %c\n", b->dim[0], b->dim[1], bli_obj_elem_size(*a ), bli_obj_row_off(*a), bli_obj_col_off(*a), b->rs, b->cs, bli_obj_has_trans(*b) ? 'Y' : 'N'); +#ifdef PRINT_SMALL_TRSM + //bli_printm("b", b, "%4.1f", ""); +#endif + fflush(stdout); +#endif +#if 0 +for (i = 0; i < m; i++) //no. of cols of B +{ + for (j = 0; j < n; j++) //no. of rows of B + { + B[i*n + j] = 1001 + j + (i*n); + } +} +for (i = 0; i < m; i++) //no. of cols of B +{ + for (j = i; j < m; j++) //no. of rows of B + { + L[i*m + j] = 2001 + j + (i*m); + } +} +#endif +#ifdef BLIS_ENABLE_SMALL_MATRIX_TRSM + gint_t status = bli_trsm_small( side, alpha, a, b, cntx, cntl ); + if ( status == BLIS_SUCCESS ) return; +#endif // Check parameters. if ( bli_error_checking_is_enabled() ) diff --git a/frame/3/trsm/bli_trsm_front.h b/frame/3/trsm/bli_trsm_front.h index 84feef22f..4e0e3cd63 100644 --- a/frame/3/trsm/bli_trsm_front.h +++ b/frame/3/trsm/bli_trsm_front.h @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018, Advanced Micro Devices, Inc Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -41,3 +42,13 @@ void bli_trsm_front cntx_t* cntx, cntl_t* cntl ); + +err_t bli_trsm_small + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); diff --git a/kernels/zen/3/bli_trsm_small.c b/kernels/zen/3/bli_trsm_small.c new file mode 100644 index 000000000..dd7dc3a36 --- /dev/null +++ b/kernels/zen/3/bli_trsm_small.c @@ -0,0 +1,5487 @@ +/* + +BLIS +An object-based framework for developing high-performance BLAS-like +libraries. + +Copyright (C) 2018, Advanced Micro Devices, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of The University of Texas at Austin nor the names +of its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ +//#define BLIS_ENABLE_SMALL_MATRIX_TRSM + +#include "blis.h" +#ifdef BLIS_ENABLE_SMALL_MATRIX_TRSM +#include "immintrin.h" + +static void (*fp_blis_strsm_microkernel)( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); +static void blis_strsm_microkernel( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); +static void blis_strsm_microkernel_alpha( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal + ); +static void blis_strsm_microkernel_unitDiag( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); +static void blis_strsm_microkernel_alpha_unitDiag( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal + ); +static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal); +static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal); + +//AX = B; A is lower triangular; No transpose; single precision +static err_t bli_strsm_small_AlXB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//AX = B; A is lower triangular; No transpose; double precision +static err_t bli_dtrsm_small_AlXB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//A.'X = B; A is upper triangular; A has to be transposed; single precision +static err_t bli_strsm_small_AutXB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//A.'X = B; A is upper triangular; A has to be transposed; double precision +static err_t bli_dtrsm_small_AutXB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//XA.' = B; A is lower triangular; A has to be transposed; single precision +static err_t bli_strsm_small_XAltB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//XA.' = B; A is lower triangular; A has to be transposed; double precision +static err_t bli_dtrsm_small_XAltB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + void trsm_block_c(float *ptr_l, float *ptr_b, int blk_height, int blk_width, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b); +/* +* The bli_trsm_small implements unpacked version of TRSM +* Currently only column-major is supported, A & B are column-major +* Input: A: MxM (triangular matrix) +* B: MxN matrix +* Output: X: MxN matrix such that AX = alpha*B or XA = alpha*B or A'X = alpha*B or XA' = alpha*B +* Here the output X is stored in B +* The custom-kernel will be called only when M*(M+N)* sizeof(Matrix Elements) < L3 cache +*/ +err_t bli_trsm_small + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ +#ifdef BLIS_ENABLE_MULTITHREADING + return BLIS_NOT_YET_IMPLEMENTED; +#endif + + // If alpha is zero, B matrix will become zero after scaling & hence solution is also zero matrix + if (bli_obj_equals(alpha, &BLIS_ZERO)) + { + return BLIS_NOT_YET_IMPLEMENTED; // scale B by alpha + } + // We have to call matrix scaling if alpha != 1.0 + + // if row major format return. Check this again. + if ((bli_obj_row_stride(*a) != 1) || + (bli_obj_row_stride(*b) != 1)) + { + return BLIS_INVALID_ROW_STRIDE; + } + + num_t dt = ((*b).info & (0x7 << 0)); + + // only float and double datatypes are supported as of now. + if (dt != BLIS_DOUBLE && dt != BLIS_FLOAT) + { + return BLIS_EXPECTED_REAL_DATATYPE; + } + + // A is expected to be triangular in trsm + if (!bli_obj_is_upper_or_lower (*a)) + { + return BLIS_EXPECTED_TRIANGULAR_OBJECT; + } + + // can use other control structs - even can use array of function pointers, + // indexed by a number with bits formed by f('side', 'uplo', 'transa', dt). + // In the below implementation, based on the number of finally implemented + // cases, can move the checks with more cases higher up. + if (side == BLIS_LEFT) + { + if (bli_obj_has_trans(*a)) + { + if (dt == BLIS_DOUBLE) + { + if (bli_obj_is_upper(*a)) + { + //A.'X = B; A is upper triangular; A has to be transposed; double precision +#if 0 // planning to implement this in this iteration + return bli_dtrsm_small_AutXB(side, alpha, a, b, cntx, cntl); +#else + return BLIS_NOT_YET_IMPLEMENTED; +#endif + } + else + { + return BLIS_NOT_YET_IMPLEMENTED; + } + } + else if (dt == BLIS_FLOAT) + { + if (bli_obj_is_upper(*a)) + { + //A.'X = B; A is upper triangular; A has to be transposed; single precision + //return bli_strsm_small_AutXB(side, alpha, a, b, cntx, cntl); + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + return BLIS_NOT_YET_IMPLEMENTED; + } + } + } + else + { + if (dt == BLIS_DOUBLE) + { + if (bli_obj_is_upper(*a)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //AX = B; A is lower triangular; No transpose; double precision + //return bli_dtrsm_small_AlXB(side, alpha, a, b, cntx, cntl); + return BLIS_NOT_YET_IMPLEMENTED; + } + } + else if (dt == BLIS_FLOAT) + { + if (bli_obj_is_upper(*a)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //AX = B; A is lower triangular; No transpose; single precision + return bli_strsm_small_AlXB(side, alpha, a, b, cntx, cntl); + } + } + } + } + else + { + if (bli_obj_has_trans(*a)) + { + if (dt == BLIS_DOUBLE) + { + if (bli_obj_is_upper(*a)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //XA.' = B; A is lower triangular; A has to be transposed; double precision +#if 0 // planning to implement this in this iteration + return bli_dtrsm_small_XAltB(side, alpha, a, b, cntx, cntl); +#else + return BLIS_NOT_YET_IMPLEMENTED; +#endif + } + } + else if (dt == BLIS_FLOAT) + { + if (bli_obj_is_upper(*a)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //XA.' = B; A is lower triangular; A has to be transposed; single precision + return bli_strsm_small_XAltB(side, alpha, a, b, cntx, cntl); + } + } + } + else + { + return BLIS_NOT_YET_IMPLEMENTED; + } + } + + return BLIS_NOT_YET_IMPLEMENTED; +}; + + + + +/* + * AX = alpha*B, Double precision, A: lower triangular + */ +static err_t bli_dtrsm_small_AlXB ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + int M = bli_obj_length(*b); // number of rows of matrix B + int N = bli_obj_width(*b); // number of columns of matrix B + + int lda = bli_obj_col_stride(*a); // column stride of A + int ldb = bli_obj_col_stride(*b); // column stride of B + + int i; + int j; + int k; + + double *A = a->buffer; + double *B = b->buffer; + + // Need to incorporate alpha + + #if 0 + + for (k = 0; k < M; k++) + { + double lkk_inv = 1.0/A[k+k*lda]; + + for (j = 0; j < N; j++) + { + B[k + j*ldb] *= lkk_inv; + } + for (i = k+1; i < M; i++) + { + for (j = 0; j < N; j++) + { + B[i + j*ldb] -= A[i + k*lda] * B[k + j*ldb]; + } + } + }// k -loop + #else + for (k = 0; k < M; k++) + { + double lkk_inv = 1.0/A[k+k*lda]; + + for (j = 0; j < N; j++) + { + B[k + j*ldb] *= lkk_inv; + + // for (j = 0; j < N; j++) + for (i = k+1; i < M; i++) + { + B[i + j*ldb] -= A[i + k*lda] * B[k + j*ldb]; + } + } + }// k -loop + + #endif + + return BLIS_SUCCESS; +}// end of function + + + +static void trsm_small_AlXB ( + float *A, + float *B, + int M, + int N, + int lda, + int ldb + ) +{ + int i; + int j; + int k; + + // Need to incorporate alpha + + for (k = 0; k < M; k++) + { + float lkk_inv = 1.0/A[k+k*lda]; + + for (j = 0; j < N; j++) + { + B[k + j*ldb] *= lkk_inv; + + for (i = k+1; i < M; i++) + { + B[i + j*ldb] -= A[i + k*lda] * B[k + j*ldb]; + } + } + }// k -loop + +}// end of function + + +// Test code: +void gemm_small( float *ptr_l, + float *ptr_b, + int blk_m, + int blk_n, + float *ptr_gemmOut, + int cs_l, + int cs_b, + int rs_l, + int rs_b, + float alpha, + float beta) +{ + int i, j, k; + + for (i = 0; i < blk_m; i++) + { + for (j = 0; j < blk_n; j++) + { + float t = 0.0; + for (k = 0; k < blk_m; k++) + { + t += (ptr_l[i*rs_l + k* cs_l] * ptr_b[k*rs_b + j*cs_b]); + } + ptr_gemmOut[i*rs_b + j*cs_b] = beta * ptr_gemmOut[i*rs_b + j*cs_b] + alpha * t; + } + } +} + + + + +/* + * AX = Alpha*B, Single precision, A: lower triangular + */ +static err_t bli_strsm_small_AlXB ( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + obj_t alpha, beta; // gemm parameters + obj_t Ga, Gb, Gc; // for GEMM + int m = bli_obj_length(*b); // number of rows of matrix B + int n = bli_obj_width(*b); // number of columns of matrix B + + int lda = bli_obj_col_stride(*a); // column stride of A + int ldb = bli_obj_col_stride(*b); // column stride of B + + int rsa = bli_obj_row_stride(*a); // row stride of A + int rsb = bli_obj_row_stride(*b); // row stride of B + + int i = 0; + int j; + int blk_size = 8; + int isUnitDiag = bli_obj_has_unit_diag(*a); + + float alphaVal; + float *L = a->buffer; + float *B = b->buffer; + + if (m != 16 || (n%8) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, *AlphaObj)); + + /* Small _GEMM preparation code */ + bli_obj_create( BLIS_FLOAT, 1, 1, 0, 0, &alpha ); + bli_obj_create( BLIS_FLOAT, 1, 1, 0, 0, &beta ); + + /* B = B - A*B */ + bli_setsc( -(1.0), 0.0, &alpha ); + bli_setsc( (1.0), 0.0, &beta ); + + + bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, blk_size, a->buffer, rsa, lda, &Ga); + bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, n, b->buffer, rsb, ldb, &Gb); + bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, n, b->buffer, rsb, ldb, &Gc); + + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, Ga ); + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, Gb ); + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, Gc ); + + //first block of trsm + Gb.buffer = (void*)(B + i); + + //trsm of first 8xn block + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + blis_strsm_microkernel_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_strsm_microkernel = blis_strsm_microkernel; + } + else + { + blis_strsm_microkernel_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; + } + bli_setsc( alphaVal, 0.0, &beta ); + } + else + { + if (isUnitDiag == 0) + { + blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_strsm_microkernel = blis_strsm_microkernel; + } + else + { + blis_strsm_microkernel_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; + } + } + + //gemm update + for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + { + Ga.buffer = (void*)(L + j + i*lda); + Gc.buffer = (void*)(B + j); + + bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + } + + //trsm of remaining blocks + for (i = blk_size; i < m; i += blk_size) + { + Gb.buffer = (void*)(B + i); + + fp_blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + + for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + { + Ga.buffer = (void*)(L + j + i*lda); + Gc.buffer = (void*)(B + j); + + bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + } + + } // End of for loop - i + + return BLIS_SUCCESS; +} + +void trsm_block_c(float *ptr_l, float *ptr_b, int blk_height, int blk_width, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + int i, j, k, l; + float inv_l; + + inv_l = 1.0 / *ptr_l; + + for (j = 0; j < numCols_b; j += blk_width) + { + for (l = j; l < (j+blk_width); l++) + { + ptr_b[l*cs_b] = ptr_b[l*cs_b] * inv_l; + } + + for (i = 1; i < blk_height; i++) + { + for (l = j; l < (j+blk_width); l++) + { + for (k = 0; k < i; k++) + { + ptr_b[i*rs_b + l*cs_b] -= (ptr_b[k*rs_b + l*cs_b] * ptr_l[i*rs_l + k*cs_l]); + } + ptr_b[i*rs_b + l*cs_b] = ptr_b[i*rs_b + l*cs_b] / ptr_l[i*rs_l + i*cs_l]; + } + } + } +} + +/* + * XA' = Alpha*B, Single precision, A: lower triangular + */ +static err_t bli_strsm_small_XAltB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int m = bli_obj_length(*a); // number of rows of matrix B + int n = bli_obj_length(*b); // number of columns of matrix B + + int lda = bli_obj_col_stride(*a); // column stride of A + int ldb = bli_obj_col_stride(*b); // column stride of B + + int rsa = bli_obj_row_stride(*a); // row stride of A + int rsb = bli_obj_row_stride(*b); // row stride of B + + int i = 0; + int isUnitDiag = bli_obj_has_unit_diag(*a); + + float alphaVal; + float *L = a->buffer; + float *B = b->buffer; + + if ((m%8) != 0 || (n%8) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, *AlphaObj)); + + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + trsm_XAtB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + else + { + trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + } + else + { + if (isUnitDiag == 0) + { + trsm_XAtB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + else + { + trsm_XAtB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + } + return BLIS_SUCCESS; +} + +static void blis_strsm_microkernel_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) +{ + float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags; + __m256 alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols +} + +static void blis_strsm_microkernel_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) +{ + //float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags; + __m256 alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //8th col + //ptr_l += cs_l; + //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols +} + +static void blis_strsm_microkernel_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //8th col + //ptr_l += cs_l; + //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols +} + +static void blis_strsm_microkernel(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols +} + +///////////////////////////////////// XA'=B functions //////////////////////////////// + +static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + __m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_cols_rearr[8] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_cols_rearr[9] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[10] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[11] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_cols_rearr[12] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_cols_rearr[13] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_cols_rearr[14] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_cols_rearr[15] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[8], mat_a_cols_rearr[9], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[10], mat_a_cols_rearr[11], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[12], mat_a_cols_rearr[13], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[15], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + __m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); + mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); + mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); + mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); + mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_cols_rearr[8] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_cols_rearr[9] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[10] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[11] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_cols_rearr[12] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_cols_rearr[13] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_cols_rearr[14] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_cols_rearr[15] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[8], mat_a_cols_rearr[9], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[10], mat_a_cols_rearr[11], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[12], mat_a_cols_rearr[13], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[15], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); + mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); + mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); + mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); + mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); + + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + //__m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0) + mat_b_col[0] = mat_b_rearr[0][0]; + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): already done + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + //__m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); + mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); + mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); + mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); + mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); + + //(Row0) + mat_b_col[0] = mat_b_rearr[0][0]; + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); + mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); + mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); + mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); + mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); + + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): already done + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} + + +#endif + From 695cd520e2f5eab938f66afe9fe36201ab2700c5 Mon Sep 17 00:00:00 2001 From: sraut Date: Wed, 6 Jun 2018 11:48:56 +0530 Subject: [PATCH 07/53] AMD Copyright information changed to 2018 Change-Id: Idfd11afd5d252f8063d0158680d24bf7e2854469 --- LICENSE | 2 +- build/auto-detect/old/cpuid_x86.c | 2 +- build/templates/license.c | 2 +- build/templates/license.h | 2 +- build/templates/license.sh | 2 +- config/zen/bli_kernel.h | 2 +- frame/3/gemm/bli_gemm_front.c | 2 +- frame/base/bli_init.c | 2 +- frame/include/bli_macro_defs.h | 2 +- frame/ind/bli_l3_ind.c | 2 +- frame/util/bli_util_unb_var1.c | 2 +- kernels/zen/1/bli_amaxv_zen_int.c | 2 +- kernels/zen/1/bli_axpyv_zen_int.c | 2 +- kernels/zen/1/bli_axpyv_zen_int10.c | 2 +- kernels/zen/1/bli_dotv_zen_int.c | 2 +- kernels/zen/1/bli_dotv_zen_int10.c | 2 +- kernels/zen/1/bli_dotxv_zen_int.c | 2 +- kernels/zen/1/bli_scalv_zen_int.c | 2 +- kernels/zen/1/bli_scalv_zen_int10.c | 2 +- kernels/zen/1f/bli_axpyf_zen_int_8.c | 2 +- kernels/zen/1f/bli_dotxf_zen_int_8.c | 2 +- kernels/zen/3/bli_gemm_zen_asm_d6x8.c | 2 +- kernels/zen/3/bli_gemmtrsm_l_zen_asm_d6x8.c | 2 +- kernels/zen/3/bli_gemmtrsm_u_zen_asm_d6x8.c | 2 +- test/Makefile | 2 +- test/test_axpyv.c | 2 +- test/test_dotv.c | 2 +- 27 files changed, 27 insertions(+), 27 deletions(-) diff --git a/LICENSE b/LICENSE index 452702ff9..340d76e5c 100644 --- a/LICENSE +++ b/LICENSE @@ -12,7 +12,7 @@ copyright info. All parties provide their portions of the code under the --- -Copyright (C) 2017, Advanced Micro Devices, Inc. +Copyright (C) 2018, Advanced Micro Devices, Inc. Copyright (C) 2014, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/build/auto-detect/old/cpuid_x86.c b/build/auto-detect/old/cpuid_x86.c index 5974453b6..095a62a01 100644 --- a/build/auto-detect/old/cpuid_x86.c +++ b/build/auto-detect/old/cpuid_x86.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2015, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/build/templates/license.c b/build/templates/license.c index c32bf75aa..3409da213 100644 --- a/build/templates/license.c +++ b/build/templates/license.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/build/templates/license.h b/build/templates/license.h index c32bf75aa..3409da213 100644 --- a/build/templates/license.h +++ b/build/templates/license.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/build/templates/license.sh b/build/templates/license.sh index 047422800..86a7d18a0 100644 --- a/build/templates/license.sh +++ b/build/templates/license.sh @@ -5,7 +5,7 @@ # libraries. # # Copyright (C) 2014, The University of Texas at Austin -# Copyright (C) 2017, Advanced Micro Devices, Inc. +# Copyright (C) 2018, Advanced Micro Devices, Inc. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are diff --git a/config/zen/bli_kernel.h b/config/zen/bli_kernel.h index 072d6104a..17a2e8277 100644 --- a/config/zen/bli_kernel.h +++ b/config/zen/bli_kernel.h @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc + Copyright (C) 2018, Advanced Micro Devices, Inc Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_front.c b/frame/3/gemm/bli_gemm_front.c index 5ae9945ab..97bb602bd 100644 --- a/frame/3/gemm/bli_gemm_front.c +++ b/frame/3/gemm/bli_gemm_front.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_init.c b/frame/base/bli_init.c index 07a9dd2bf..131d971cf 100644 --- a/frame/base/bli_init.c +++ b/frame/base/bli_init.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/include/bli_macro_defs.h b/frame/include/bli_macro_defs.h index 5409082b3..fd4f77136 100644 --- a/frame/include/bli_macro_defs.h +++ b/frame/include/bli_macro_defs.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/ind/bli_l3_ind.c b/frame/ind/bli_l3_ind.c index 20146cd9b..7c6f62950 100644 --- a/frame/ind/bli_l3_ind.c +++ b/frame/ind/bli_l3_ind.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/util/bli_util_unb_var1.c b/frame/util/bli_util_unb_var1.c index 6f499524b..16906ce0c 100644 --- a/frame/util/bli_util_unb_var1.c +++ b/frame/util/bli_util_unb_var1.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/1/bli_amaxv_zen_int.c b/kernels/zen/1/bli_amaxv_zen_int.c index af001dad8..cca28288e 100644 --- a/kernels/zen/1/bli_amaxv_zen_int.c +++ b/kernels/zen/1/bli_amaxv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016, Advanced Micro Devices, Inc + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/1/bli_axpyv_zen_int.c b/kernels/zen/1/bli_axpyv_zen_int.c index d84fed5af..6132cbad2 100644 --- a/kernels/zen/1/bli_axpyv_zen_int.c +++ b/kernels/zen/1/bli_axpyv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/1/bli_axpyv_zen_int10.c b/kernels/zen/1/bli_axpyv_zen_int10.c index a6909ebcd..4d69c2127 100644 --- a/kernels/zen/1/bli_axpyv_zen_int10.c +++ b/kernels/zen/1/bli_axpyv_zen_int10.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/1/bli_dotv_zen_int.c b/kernels/zen/1/bli_dotv_zen_int.c index 71a7caa8c..93c9df23f 100644 --- a/kernels/zen/1/bli_dotv_zen_int.c +++ b/kernels/zen/1/bli_dotv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/1/bli_dotv_zen_int10.c b/kernels/zen/1/bli_dotv_zen_int10.c index 0b8e7f729..f905ca67a 100644 --- a/kernels/zen/1/bli_dotv_zen_int10.c +++ b/kernels/zen/1/bli_dotv_zen_int10.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/1/bli_dotxv_zen_int.c b/kernels/zen/1/bli_dotxv_zen_int.c index cd8fa6816..53ca36653 100644 --- a/kernels/zen/1/bli_dotxv_zen_int.c +++ b/kernels/zen/1/bli_dotxv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/1/bli_scalv_zen_int.c b/kernels/zen/1/bli_scalv_zen_int.c index 29bcea478..dffb8059c 100644 --- a/kernels/zen/1/bli_scalv_zen_int.c +++ b/kernels/zen/1/bli_scalv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/1/bli_scalv_zen_int10.c b/kernels/zen/1/bli_scalv_zen_int10.c index 4149309d9..26ea33301 100644 --- a/kernels/zen/1/bli_scalv_zen_int10.c +++ b/kernels/zen/1/bli_scalv_zen_int10.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/1f/bli_axpyf_zen_int_8.c b/kernels/zen/1f/bli_axpyf_zen_int_8.c index 3e892f155..46e7a75e6 100644 --- a/kernels/zen/1f/bli_axpyf_zen_int_8.c +++ b/kernels/zen/1f/bli_axpyf_zen_int_8.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/1f/bli_dotxf_zen_int_8.c b/kernels/zen/1f/bli_dotxf_zen_int_8.c index 7de80d24f..0507ff97d 100644 --- a/kernels/zen/1f/bli_dotxf_zen_int_8.c +++ b/kernels/zen/1f/bli_dotxf_zen_int_8.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/3/bli_gemm_zen_asm_d6x8.c b/kernels/zen/3/bli_gemm_zen_asm_d6x8.c index accc70af1..e70851300 100644 --- a/kernels/zen/3/bli_gemm_zen_asm_d6x8.c +++ b/kernels/zen/3/bli_gemm_zen_asm_d6x8.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/3/bli_gemmtrsm_l_zen_asm_d6x8.c b/kernels/zen/3/bli_gemmtrsm_l_zen_asm_d6x8.c index 73c196289..8685078fe 100644 --- a/kernels/zen/3/bli_gemmtrsm_l_zen_asm_d6x8.c +++ b/kernels/zen/3/bli_gemmtrsm_l_zen_asm_d6x8.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/3/bli_gemmtrsm_u_zen_asm_d6x8.c b/kernels/zen/3/bli_gemmtrsm_u_zen_asm_d6x8.c index 0e3c3f1a9..169954aa0 100644 --- a/kernels/zen/3/bli_gemmtrsm_u_zen_asm_d6x8.c +++ b/kernels/zen/3/bli_gemmtrsm_u_zen_asm_d6x8.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/Makefile b/test/Makefile index 7ecaee1b7..72736d576 100644 --- a/test/Makefile +++ b/test/Makefile @@ -5,7 +5,7 @@ # libraries. # # Copyright (C) 2014, The University of Texas at Austin -# Copyright (C) 2017, Advanced Micro Devices, Inc. +# Copyright (C) 2018, Advanced Micro Devices, Inc. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are diff --git a/test/test_axpyv.c b/test/test_axpyv.c index 2e2e5928c..e56841aa9 100644 --- a/test/test_axpyv.c +++ b/test/test_axpyv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/test_dotv.c b/test/test_dotv.c index 4f93df5d6..93547064b 100644 --- a/test/test_dotv.c +++ b/test/test_dotv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are From 7af5283dcc3dded114852d6013d33134021b81aa Mon Sep 17 00:00:00 2001 From: sraut Date: Mon, 11 Jun 2018 15:00:22 +0530 Subject: [PATCH 08/53] added check condition on n-dimension for XA'=B intrinsic code to process till 128 size Change-Id: I95d020a5ca3ea21d446b8c2e379d56e1eea18530 --- kernels/zen/3/bli_trsm_small.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernels/zen/3/bli_trsm_small.c b/kernels/zen/3/bli_trsm_small.c index dd7dc3a36..7c2931b07 100644 --- a/kernels/zen/3/bli_trsm_small.c +++ b/kernels/zen/3/bli_trsm_small.c @@ -664,7 +664,7 @@ static err_t bli_strsm_small_XAltB( { return BLIS_NOT_YET_IMPLEMENTED; } - if ( (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + if ( n > 128 || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) { return BLIS_NOT_YET_IMPLEMENTED; } From 9814cfdf3157ef4726ee604fc895d56e8063d765 Mon Sep 17 00:00:00 2001 From: Meghana Date: Fri, 28 Sep 2018 11:02:39 +0530 Subject: [PATCH 09/53] fixed blastest level-3 failure by adding ((M&N&K) != 0) to check condition in bli_gemm_small.c Change-Id: I85e4a32996ebb880f3c00bd293edc38f74700fe6 --- kernels/zen/3/bli_gemm_small.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernels/zen/3/bli_gemm_small.c b/kernels/zen/3/bli_gemm_small.c index 1a1ac9b26..921f57bb2 100644 --- a/kernels/zen/3/bli_gemm_small.c +++ b/kernels/zen/3/bli_gemm_small.c @@ -178,8 +178,8 @@ static err_t bli_sgemm_small int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . // printf("alpha_cast = %f beta_cast = %f [ Trans = %d %d], [stride = %d %d %d] [m,n,k = %d %d %d]\n",*alpha_cast,*beta_cast, bli_obj_has_trans( a ), bli_obj_has_trans( b ), lda, ldb,ldc, M,N,K); - if (((M * N) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES)) - || ((M < BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < BLIS_SMALL_K_RECT_MATRIX_THRES))) + if ((((M * N) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES)) + || ((M < BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((M&N&K)!=0)) { int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. @@ -1580,8 +1580,8 @@ static err_t bli_dgemm_small // If alpha is zero, scale by beta and return. // printf("alpha_cast = %f beta_cast = %f [ Trans = %d %d], [stride = %d %d %d] [m,n,k = %d %d %d]\n",*alpha_cast,*beta_cast, bli_obj_has_trans( a ), bli_obj_has_trans( b ), lda, ldb,ldc, M,N,K); - if (((M * N) < (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES)) - || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) + if ((((M * N) < (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES)) + || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((M&N&K) != 0)) { int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. From 08045a6c52b6e025652c5b18eb120c0f4e61cf6f Mon Sep 17 00:00:00 2001 From: sraut Date: Mon, 1 Oct 2018 15:38:23 +0530 Subject: [PATCH 10/53] Corrected the fix made for blastest level-3 failure to check m,n,k non-zero condition in bli_gemm_small.c Change-Id: Idaf9f2327c3127b04a2738ae8a058b83d6c57934 --- kernels/zen/3/bli_gemm_small.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kernels/zen/3/bli_gemm_small.c b/kernels/zen/3/bli_gemm_small.c index 921f57bb2..ce9926fbb 100644 --- a/kernels/zen/3/bli_gemm_small.c +++ b/kernels/zen/3/bli_gemm_small.c @@ -176,10 +176,11 @@ static err_t bli_sgemm_small int M = bli_obj_length( c ); // number of rows of Matrix C int N = bli_obj_width( c ); // number of columns of Matrix C int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + int L = M * N; // printf("alpha_cast = %f beta_cast = %f [ Trans = %d %d], [stride = %d %d %d] [m,n,k = %d %d %d]\n",*alpha_cast,*beta_cast, bli_obj_has_trans( a ), bli_obj_has_trans( b ), lda, ldb,ldc, M,N,K); - if ((((M * N) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES)) - || ((M < BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((M&N&K)!=0)) + if ((((L) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES)) + || ((M < BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) { int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. @@ -1577,11 +1578,12 @@ static err_t bli_dgemm_small int M = bli_obj_length( c ); // number of rows of Matrix C int N = bli_obj_width( c ); // number of columns of Matrix C int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + int L = M * N; // If alpha is zero, scale by beta and return. // printf("alpha_cast = %f beta_cast = %f [ Trans = %d %d], [stride = %d %d %d] [m,n,k = %d %d %d]\n",*alpha_cast,*beta_cast, bli_obj_has_trans( a ), bli_obj_has_trans( b ), lda, ldb,ldc, M,N,K); - if ((((M * N) < (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES)) - || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((M&N&K) != 0)) + if ((((L) < (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES)) + || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) { int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. From ee46fa3efb6e920fa6c3d0b0601007f5de31deb5 Mon Sep 17 00:00:00 2001 From: sraut Date: Mon, 1 Oct 2018 16:30:30 +0530 Subject: [PATCH 11/53] Small TRSM optimization changes :- 1) single precision small trsm kernels for XAt=B case are further optimized for performance. 2) double precision small trsm kernels for AX=B and XAtB cases are implemented. 3) single precision small trsm kernels for AutX=B are implemented in intrinsics to improve the current performance. Change-Id: Ic9d67ae6d8522615257dde018903f049dcffa2cf --- kernels/zen/3/bli_trsm_small.c | 9885 +++++++++++++++++++++++++++++++- 1 file changed, 9753 insertions(+), 132 deletions(-) diff --git a/kernels/zen/3/bli_trsm_small.c b/kernels/zen/3/bli_trsm_small.c index d18e2da14..a64db0e2d 100644 --- a/kernels/zen/3/bli_trsm_small.c +++ b/kernels/zen/3/bli_trsm_small.c @@ -26,17 +26,21 @@ HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -//#define BLIS_ENABLE_SMALL_MATRIX_TRSM #include "blis.h" #ifdef BLIS_ENABLE_SMALL_MATRIX_TRSM #include "immintrin.h" +#define GEMM_BLK_V1 8 +#define GEMM_ACCUM_A 1 +#define OPT_CACHE_BLOCKING_L1 1 +#define REARRANGE_SHFL 0 + static void (*fp_blis_strsm_microkernel)( float *ptr_l, float *ptr_b, int numRows_lb, @@ -119,6 +123,127 @@ static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, int cs_b, float alphaVal); +static void (*fp_blis_dtrsm_microkernel)( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); + +static void blis_dtrsm_microkernel( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); + +static void blis_dtrsm_microkernel_alpha( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ); + +static void blis_dtrsm_microkernel_unitDiag( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); + +static void blis_dtrsm_microkernel_alpha_unitDiag( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ); + +static void dtrsm_XAtB_block_allSmallSizedMatrices(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal); +static void dtrsm_XAtB_block_allSmallSizedMatrices_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal); +static void trsm_AutXB_block_allSmallSizedMatrices(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void trsm_AutXB_block_allSmallSizedMatrices_alpha(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alpha); +static void trsm_AutXB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alpha); + //AX = B; A is lower triangular; No transpose; single precision static err_t bli_strsm_small_AlXB ( @@ -129,18 +254,6 @@ static err_t bli_strsm_small_AlXB cntx_t* cntx, cntl_t* cntl ); - -//AX = B; A is lower triangular; No transpose; double precision -static err_t bli_dtrsm_small_AlXB - ( - side_t side, - obj_t* alpha, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ); - //A.'X = B; A is upper triangular; A has to be transposed; single precision static err_t bli_strsm_small_AutXB ( @@ -152,17 +265,6 @@ static err_t bli_strsm_small_AutXB cntl_t* cntl ); -//A.'X = B; A is upper triangular; A has to be transposed; double precision -static err_t bli_dtrsm_small_AutXB - ( - side_t side, - obj_t* alpha, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ); - //XA.' = B; A is lower triangular; A has to be transposed; single precision static err_t bli_strsm_small_XAltB ( @@ -173,6 +275,29 @@ static err_t bli_strsm_small_XAltB cntx_t* cntx, cntl_t* cntl ); +//AX = B; A is lower triangular; No transpose; double precision +static err_t bli_dtrsm_small_AlXB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + + +//A.'X = B; A is upper triangular; A has to be transposed; double precision +static err_t bli_dtrsm_small_AutXB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + //XA.' = B; A is lower triangular; A has to be transposed; double precision static err_t bli_dtrsm_small_XAltB @@ -265,8 +390,8 @@ err_t bli_trsm_small if (bli_obj_is_upper(a)) { //A.'X = B; A is upper triangular; A has to be transposed; single precision - //return bli_strsm_small_AutXB(side, alpha, a, b, cntx, cntl); - return BLIS_NOT_YET_IMPLEMENTED; + return bli_strsm_small_AutXB(side, alpha, a, b, cntx, cntl); + //return BLIS_NOT_YET_IMPLEMENTED; } else { @@ -285,8 +410,8 @@ err_t bli_trsm_small else { //AX = B; A is lower triangular; No transpose; double precision - //return bli_dtrsm_small_AlXB(side, alpha, a, b, cntx, cntl); - return BLIS_NOT_YET_IMPLEMENTED; + return bli_dtrsm_small_AlXB(side, alpha, a, b, cntx, cntl); + //return BLIS_NOT_YET_IMPLEMENTED; } } else if (dt == BLIS_FLOAT) @@ -299,6 +424,7 @@ err_t bli_trsm_small { //AX = B; A is lower triangular; No transpose; single precision return bli_strsm_small_AlXB(side, alpha, a, b, cntx, cntl); + //return BLIS_NOT_YET_IMPLEMENTED; } } } @@ -316,12 +442,9 @@ err_t bli_trsm_small else { //XA.' = B; A is lower triangular; A has to be transposed; double precision -#if 0 // planning to implement this in this iteration - return bli_dtrsm_small_XAltB(side, alpha, a, b, cntx, cntl); -#else - return BLIS_NOT_YET_IMPLEMENTED; -#endif - } + return bli_dtrsm_small_XAltB(side, alpha, a, b, cntx, cntl); + //return BLIS_NOT_YET_IMPLEMENTED; + } } else if (dt == BLIS_FLOAT) { @@ -332,6 +455,7 @@ err_t bli_trsm_small else { //XA.' = B; A is lower triangular; A has to be transposed; single precision + //return BLIS_NOT_YET_IMPLEMENTED; return bli_strsm_small_XAltB(side, alpha, a, b, cntx, cntl); } } @@ -346,78 +470,6 @@ err_t bli_trsm_small }; - - -/* - * AX = alpha*B, Double precision, A: lower triangular - */ -static err_t bli_dtrsm_small_AlXB ( - side_t side, - obj_t* alpha, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - - int M = bli_obj_length(b); // number of rows of matrix B - int N = bli_obj_width(b); // number of columns of matrix B - - int lda = bli_obj_col_stride(a); // column stride of A - int ldb = bli_obj_col_stride(b); // column stride of B - - int i; - int j; - int k; - - double *A = a->buffer; - double *B = b->buffer; - - // Need to incorporate alpha - - #if 0 - - for (k = 0; k < M; k++) - { - double lkk_inv = 1.0/A[k+k*lda]; - - for (j = 0; j < N; j++) - { - B[k + j*ldb] *= lkk_inv; - } - for (i = k+1; i < M; i++) - { - for (j = 0; j < N; j++) - { - B[i + j*ldb] -= A[i + k*lda] * B[k + j*ldb]; - } - } - }// k -loop - #else - for (k = 0; k < M; k++) - { - double lkk_inv = 1.0/A[k+k*lda]; - - for (j = 0; j < N; j++) - { - B[k + j*ldb] *= lkk_inv; - - // for (j = 0; j < N; j++) - for (i = k+1; i < M; i++) - { - B[i + j*ldb] -= A[i + k*lda] * B[k + j*ldb]; - } - } - }// k -loop - - #endif - - return BLIS_SUCCESS; -}// end of function - - - static void trsm_small_AlXB ( float *A, float *B, @@ -480,7 +532,125 @@ void gemm_small( float *ptr_l, } } +/* + * AX = Alpha*B, Double precision, A:lower triangular +SUPPORTS MATRIX SIZE OF THE FORM 16X4*i, WHERE i IS AN INTEGER + */ +static err_t bli_dtrsm_small_AlXB ( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + obj_t alpha, beta; // gemm parameters + obj_t Ga, Gb, Gc; // for GEMM + int m = bli_obj_length(b); // number of rows of matrix B + int n = bli_obj_width(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int j; + int blk_size = 4; + int isUnitDiag = bli_obj_has_unit_diag(a); + + double alphaVal; + double *L = a->buffer; + double *B = b->buffer; + + if (m !=16 || (n&3) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((double *)AlphaObj->buffer); + + /* Small _GEMM preparation code */ + bli_obj_create( BLIS_DOUBLE, 1, 1, 0, 0, &alpha ); + bli_obj_create( BLIS_DOUBLE, 1, 1, 0, 0, &beta ); + + /* B = B - A*B */ + bli_setsc( -(1.0), 0.0, &alpha ); + bli_setsc( (1.0), 0.0, &beta ); + + bli_obj_create_with_attached_buffer( BLIS_DOUBLE, blk_size, blk_size, a->buffer, rsa, lda, &Ga); + bli_obj_create_with_attached_buffer( BLIS_DOUBLE, blk_size, n, b->buffer, rsb, ldb, &Gb); + bli_obj_create_with_attached_buffer( BLIS_DOUBLE, blk_size, n, b->buffer, rsb, ldb, &Gc); + + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Ga ); + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gb ); + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gc ); + + //first block of trsm + Gb.buffer = (void*)(B + i); + + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + blis_dtrsm_microkernel_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_dtrsm_microkernel = blis_dtrsm_microkernel; + } + else + { + blis_dtrsm_microkernel_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_dtrsm_microkernel = blis_dtrsm_microkernel_unitDiag; + } + bli_setsc( alphaVal, 0.0, &beta ); + } + else + { + if (isUnitDiag == 0) + { + blis_dtrsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_dtrsm_microkernel = blis_dtrsm_microkernel; + } + else + { + blis_dtrsm_microkernel_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_dtrsm_microkernel = blis_dtrsm_microkernel_unitDiag; + } + } + + +//gemm update + for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + { + Ga.buffer = (void*)(L + j + i*lda); + Gc.buffer = (void*)(B + j); + bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + } + bli_setsc( (1.0), 0.0, &beta ); + + //trsm of remaining blocks + for (i = blk_size; i < m; i += blk_size) + { + Gb.buffer = (void*)(B + i); + + fp_blis_dtrsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + + + for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + { + Ga.buffer = (void*)(L + j + i*lda); + Gc.buffer = (void*)(B + j); + + bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + } + + } // End of for loop - i + + return BLIS_SUCCESS; + +} /* @@ -632,6 +802,73 @@ void trsm_block_c(float *ptr_l, float *ptr_b, int blk_height, int blk_width, int } } + +/* + * XA' = Alpha*B, Double precision, A:lower triangular + */ + +static err_t bli_dtrsm_small_XAltB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + int m = bli_obj_length(a); // number of rows of matrix B + int n = bli_obj_length(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int isUnitDiag = bli_obj_has_unit_diag(a); + + double alphaVal; + double *L = a->buffer; + double *B = b->buffer; + + if ((m%4) != 0 || (n%4) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( n > 64 || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + alphaVal = *((double *)AlphaObj->buffer); + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + dtrsm_XAtB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + else + { + dtrsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + } + else + { + if (isUnitDiag == 0) + { + dtrsm_XAtB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + else + { + dtrsm_XAtB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + } + return BLIS_SUCCESS; + +} + + /* * XA' = Alpha*B, Single precision, A: lower triangular */ @@ -696,6 +933,974 @@ static err_t bli_strsm_small_XAltB( return BLIS_SUCCESS; } +/* + * A'X = Alpha*B, Single precision, A: upper triangular + */ +static err_t bli_strsm_small_AutXB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int m = bli_obj_width(a); // number of rows of matrix A (since At, so width is taken) + int n = bli_obj_width(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int isUnitDiag = bli_obj_has_unit_diag(a); + + float alphaVal; + float *L = a->buffer; + float *B = b->buffer; + + if ((m%8) != 0 || (n%8) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( m > 64 || n > 128 || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); + + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + trsm_AutXB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + else + { + trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + } + else + { + if (isUnitDiag == 0) + { + trsm_AutXB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + else + { + trsm_AutXB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + } + return BLIS_SUCCESS; +} +/* +* AX=B A=LOWER TRIANGULAR, NO TRANSPOSE, NON-UNITDIAGONAL +* ALPHA != 1; +*/ +static void blis_dtrsm_microkernel_alpha(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ) +{ + double ones = 1.0; + int j; + int cs_b_offset[2]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags; + __m256d alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + + reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); + alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); + + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + numCols_b -= 4; // blk_width = 4 + + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); + reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); + + for(j = 0;j < numCols_b; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + } + //Last block trsm processing + + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + +} +/* +*AX=B A=LOWER TRIANGULAR, NO TRANSPOSE, UNITDIAGONAL +*ALPHA != 1; +*/ +static void blis_dtrsm_microkernel_alpha_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ) +{ + + int j; + int cs_b_offset[2]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + __m256d alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + + alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + numCols_b -= 4; // blk_width = 4 + + for(j = 0;j < numCols_b; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + } + //Last block trsm processing + + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + + ///unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + +} +/* +*AX = B A= LOWERTRIANGULAR, NO TRANSPOSE, NON-UNITDIAGONAL +*ALPHA = 1 +*/ +static void blis_dtrsm_microkernel(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ) +{ + double ones = 1.0; + int j; + int cs_b_offset[2]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + + reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + numCols_b -= 4; // blk_width = 4 + + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); + reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); + + for(j = 0;j < numCols_b; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + } + //Last block trsm processing + + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + + +} +/* +*AX = B A=LOWER TRIANGULAR, NO TRANSPOSE, UNITDIAGONAL +*ALPHA = 1 +*/ +static void blis_dtrsm_microkernel_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ) +{ + + + //double ones = 1.0; + int j; + int cs_b_offset[2]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + numCols_b -= 4; // blk_width = 4 + + + for(j = 0;j < numCols_b; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + } + //Last block trsm processing + + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + +} +///////////////////////////// AX=B /////////////////////////////// static void blis_strsm_microkernel_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) { float ones = 1.0; @@ -3133,7 +4338,4096 @@ static void blis_strsm_microkernel(float *ptr_l, float *ptr_b, int numRows_lb, i } ///////////////////////////////////// XA'=B functions //////////////////////////////// +static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alpha + ) +{ + + double ones = 1.0; + int i,i1,i2,i3,i4,j,k,l; + int cs_b_offset[3]; + int cs_l_offset[3]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[16][4]; + __m256d mat_a_cols_rearr[4]; + __m256d mat_a_blk_elems[16]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags[2]; + __m256d alphaReg; + reciprocal_diags[0] = _mm256_broadcast_sd((double const *)(&ones)); + alphaReg = _mm256_broadcast_sd((double const *)&alpha); + + // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + + //read diag elems of L 4x4 block + mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[1]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 4 diags together + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); + + //Broadcast A32 to register + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + /***************** first set of 4 cols of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 4) + { + /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A + //read 4x4 block of B into registers + + mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + mat_b_rearr[0][0] = _mm256_mul_pd(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_pd(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_pd(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_pd(mat_b_rearr[3][0], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_pd(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_pd(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_pd(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_pd(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + + i += 4; + ptr_b_dup += 4; + + } + + /***************** first set of 4 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width} + + for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row + { + ptr_l += 4; + ptr_b_dup += cs_b_offset[2]; + i1 += cs_b_offset[2]; + //printf("i1 = i3 = %g\n",*(ptr_l+i1)); + //Read next 4x4 block of A to get diag elements + i3 += cs_l_offset[2]; + mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[1]); + + //pack 4 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 + + //reciprocal of diagnal elements of A :- 0,1,2,3 + reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 4) + { + i = i1 + k; + mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_pd(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_pd(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_pd(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_pd(mat_b_rearr[i2][3], alphaReg); + i2++; + } + + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 4) // move across m + { + + //Broadcast A4,0 to A7,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + + //Broadcast A41 to A71 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); + + //Broadcast A4,2 to A7,2 to registers + mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); + + //Broadcast A4,3 to A7,3 to registers + mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); + + i += cs_l_offset[2]; + + for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); + + + i4 = k >> 2; + + //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + + + //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + //end loop of cols + + } + i2 += cs_b_offset[2]; + + } + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + k = 0; + for (i = 0; i < numCols_b; i+=4) + { + + + + /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_pd(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_pd(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_pd(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_pd(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //Store the computed B columns + + _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + + k++; + } + } + +} + +static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alpha + ) + +{ + + int i,i1,i2,i3,i4,j,k,l; + int cs_b_offset[3]; + int cs_l_offset[3]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[16][4]; + __m256d mat_a_blk_elems[16]; + __m256d alphaReg; + alphaReg = _mm256_broadcast_sd((double const *)&alpha); + + // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); + + //Broadcast A32 to register + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); + + /***************** first set of 4 cols of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 4) + { + /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A + //read 4x4 block of B into registers + + mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + mat_b_rearr[0][0] = _mm256_mul_pd(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_pd(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_pd(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_pd(mat_b_rearr[3][0], alphaReg); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[0][0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[0][0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[0][0], mat_b_rearr[3][0]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[1][0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[1][0], mat_b_rearr[3][0]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_rearr[2][0], mat_b_rearr[3][0]);//d = c - (a*b) + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_b_rearr[0][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_rearr[1][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3][0]); + + i += 4; + ptr_b_dup += 4; + + } + + /***************** first set of 4 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width} + + for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row + { + ptr_l += 4; + ptr_b_dup += cs_b_offset[2]; + i1 += cs_b_offset[2]; + i3 += cs_l_offset[2]; + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 4) + { + i = i1 + k; + mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_pd(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_pd(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_pd(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_pd(mat_b_rearr[i2][3], alphaReg); + i2++; + } + + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 4) // move across m + { + + //Broadcast A4,0 to A7,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + + //Broadcast A41 to A71 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); + + //Broadcast A4,2 to A7,2 to registers + mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); + + //Broadcast A4,3 to A7,3 to registers + mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); + + i += cs_l_offset[2]; + + for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); + + + i4 = k >> 2; + + //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + + + //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + //end loop of cols + + } + i2 += cs_b_offset[2]; + + } + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + k = 0; + for (i = 0; i < numCols_b; i+=4) + { + + + + /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + + //Store the computed B columns + + _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + + k++; + } + + } + + +} + +static void dtrsm_XAtB_block_allSmallSizedMatrices_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ) + +{ + + int i,i1,i2,i3,i4,j,k,l; + int cs_b_offset[3]; + int cs_l_offset[3]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[16][4]; + __m256d mat_a_blk_elems[16]; + + // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); + + //Broadcast A32 to register + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); + + /***************** first set of 4 cols of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 4) + { + /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A + //read 4x4 block of B into registers + + mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[0][0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[0][0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[0][0], mat_b_rearr[3][0]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[1][0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[1][0], mat_b_rearr[3][0]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_rearr[2][0], mat_b_rearr[3][0]);//d = c - (a*b) + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_b_rearr[0][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_rearr[1][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3][0]); + + i += 4; + ptr_b_dup += 4; + + } + + /***************** first set of 4 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width} + + for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row + { + ptr_l += 4; + ptr_b_dup += cs_b_offset[2]; + i1 += cs_b_offset[2]; + i3 += cs_l_offset[2]; + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 4) + { + i = i1 + k; + mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + i2++; + } + + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 4) // move across m + { + + //Broadcast A4,0 to A7,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + + //Broadcast A41 to A71 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); + + //Broadcast A4,2 to A7,2 to registers + mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); + + //Broadcast A4,3 to A7,3 to registers + mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); + + i += cs_l_offset[2]; + + for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); + + + i4 = k >> 2; + + //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + + + //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + //end loop of cols + + } + i2 += cs_b_offset[2]; + + } + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + k = 0; + for (i = 0; i < numCols_b; i+=4) + { + + + + /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + + //Store the computed B columns + + _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + + k++; + } + + } + +} +static void dtrsm_XAtB_block_allSmallSizedMatrices(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ) + +{ + + double ones = 1.0; + int i,i1,i2,i3,i4,j,k,l; + int cs_b_offset[3]; + int cs_l_offset[3]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[16][4]; + __m256d mat_a_cols_rearr[4]; + __m256d mat_a_blk_elems[16]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_sd((double const *)(&ones)); + + // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + + //read diag elems of L 4x4 block + mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[1]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 4 diags together + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); + + //Broadcast A32 to register + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + /***************** first set of 4 cols of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 4) + { + /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A + //read 4x4 block of B into registers + + mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_pd(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_pd(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_pd(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_pd(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + + i += 4; + ptr_b_dup += 4; + + } + + /***************** first set of 4 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width} + + for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row + { + ptr_l += 4; + ptr_b_dup += cs_b_offset[2]; + i1 += cs_b_offset[2]; + //printf("i1 = i3 = %g\n",*(ptr_l+i1)); + //Read next 4x4 block of A to get diag elements + i3 += cs_l_offset[2]; + mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[1]); + + //pack 4 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 + + //reciprocal of diagnal elements of A :- 0,1,2,3 + reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 4) + { + i = i1 + k; + mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + i2++; + } + + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 4) // move across m + { + + //Broadcast A4,0 to A7,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + + //Broadcast A41 to A71 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); + + //Broadcast A4,2 to A7,2 to registers + mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); + + //Broadcast A4,3 to A7,3 to registers + mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); + + i += cs_l_offset[2]; + + for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); + + + i4 = k >> 2; + + //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + + + //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + //end loop of cols + + } + i2 += cs_b_offset[2]; + + } + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + k = 0; + for (i = 0; i < numCols_b; i+=4) + { + + + + /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_pd(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_pd(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_pd(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_pd(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //Store the computed B columns + + _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + + k++; + } + + } + +} +#if OPT_CACHE_BLOCKING_L1 //new intrinsic kernels +static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0) + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0) + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done + +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} +#else //rel 1.0 intrisic kernels (NOT OPT_CACHE_BLOCKING_L1) static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) { float ones = 1.0; @@ -3387,21 +8681,21 @@ static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, in //Read next 8x8 block of A to get diag elements i3 += cs_l_offset[6]; - mat_a_cols_rearr[8] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_cols_rearr[9] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_cols_rearr[10] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_cols_rearr[11] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_cols_rearr[12] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_cols_rearr[13] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_cols_rearr[14] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_cols_rearr[15] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); //pack 8 diags of A together reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[8], mat_a_cols_rearr[9], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[10], mat_a_cols_rearr[11], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[12], mat_a_cols_rearr[13], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[15], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 @@ -4054,21 +9348,21 @@ static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr //Read next 8x8 block of A to get diag elements i3 += cs_l_offset[6]; - mat_a_cols_rearr[8] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_cols_rearr[9] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_cols_rearr[10] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_cols_rearr[11] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_cols_rearr[12] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_cols_rearr[13] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_cols_rearr[14] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_cols_rearr[15] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); //pack 8 diags of A together reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[8], mat_a_cols_rearr[9], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[10], mat_a_cols_rearr[11], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[12], mat_a_cols_rearr[13], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[15], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 @@ -4458,7 +9752,6 @@ static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); k++; } @@ -5481,7 +10774,4335 @@ static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, f } ///////////////////loop ends ///////////////////// } +#endif //OPT_CACHE_BLOCKING_L1 + +//////////////////////////// AutX=B /////////////////////// +static void trsm_AutXB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += 8; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ +#endif + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_AutXB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += 8; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_AutXB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int i, i1, i2, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + + //(Row0) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ +#endif + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done + +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + //float ones = 1.0; + int i, i1, i2, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done + +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} +#endif From 80a8b3dd8034ec8bc03d31be3f9c837c3f6fc94b Mon Sep 17 00:00:00 2001 From: sraut Date: Wed, 3 Oct 2018 15:30:33 +0530 Subject: [PATCH 12/53] Review comments incorporated for small TRSM. Change-Id: Ia64b7b2c0375cc501c2cb0be8a1af93111808cd9 --- kernels/zen/3/bli_trsm_small.c | 24649 +++++++++++++++---------------- 1 file changed, 12281 insertions(+), 12368 deletions(-) diff --git a/kernels/zen/3/bli_trsm_small.c b/kernels/zen/3/bli_trsm_small.c index a64db0e2d..ca8e5b142 100644 --- a/kernels/zen/3/bli_trsm_small.c +++ b/kernels/zen/3/bli_trsm_small.c @@ -36,214 +36,220 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef BLIS_ENABLE_SMALL_MATRIX_TRSM #include "immintrin.h" -#define GEMM_BLK_V1 8 -#define GEMM_ACCUM_A 1 -#define OPT_CACHE_BLOCKING_L1 1 -#define REARRANGE_SHFL 0 +#define GEMM_BLK_V1 8 //Block size to perform gemm and apply trsm +#define GEMM_ACCUM_A 1 //Peform B1=B1-(B0*A0) operation instead of B1'=(B0*A0) and then B1=B1-B1' +#define OPT_CACHE_BLOCKING_L1 1 //Perform trsm block-wise in blocks of GEMM_BLK_V1 instead of all columns of B together. +#define REARRANGE_SHFL 0 //Rearrange operations using blend or shuffle +#define BLI_AlXB_M_SP 16 +#define BLI_AlXB_M_DP 16 +#define BLI_XAltB_N_SP 128 +#define BLI_XAltB_N_DP 64 +#define BLI_AutXB_M_SP 64 +#define BLI_AutXB_N_SP 128 static void (*fp_blis_strsm_microkernel)( float *ptr_l, - float *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ); + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); static void blis_strsm_microkernel( float *ptr_l, - float *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ); + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); static void blis_strsm_microkernel_alpha( float *ptr_l, - float *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - float alphaVal - ); + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal + ); static void blis_strsm_microkernel_unitDiag( float *ptr_l, - float *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ); + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); static void blis_strsm_microkernel_alpha_unitDiag( float *ptr_l, - float *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - float alphaVal - ); + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal + ); static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, - float *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b); + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, - float *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - float alphaVal); + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal); static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, - float *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b); + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, - float *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - float alphaVal); - + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal); + static void (*fp_blis_dtrsm_microkernel)( double *ptr_l, double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ); + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); static void blis_dtrsm_microkernel( double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ); + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); static void blis_dtrsm_microkernel_alpha( double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - double alphaVal - ); + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ); static void blis_dtrsm_microkernel_unitDiag( double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ); + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); static void blis_dtrsm_microkernel_alpha_unitDiag( double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - double alphaVal - ); + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ); static void dtrsm_XAtB_block_allSmallSizedMatrices(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b); + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - double alphaVal); + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal); static void dtrsm_XAtB_block_allSmallSizedMatrices_unitDiag(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b); + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - double alphaVal); + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal); static void trsm_AutXB_block_allSmallSizedMatrices(float *ptr_l, - float *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b); + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); static void trsm_AutXB_block_allSmallSizedMatrices_alpha(float *ptr_l, - float *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - float alpha); + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alpha); static void trsm_AutXB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, - float *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b); + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); static void trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, - float *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - float alpha); - + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alpha); + //AX = B; A is lower triangular; No transpose; single precision static err_t bli_strsm_small_AlXB ( @@ -309,7 +315,7 @@ static err_t bli_dtrsm_small_XAltB cntx_t* cntx, cntl_t* cntl ); - void trsm_block_c(float *ptr_l, float *ptr_b, int blk_height, int blk_width, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b); + void trsm_block_c(float *ptr_l, float *ptr_b, int blk_height, int blk_width, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b); /* * The bli_trsm_small implements unpacked version of TRSM * Currently only column-major is supported, A & B are column-major @@ -352,13 +358,13 @@ err_t bli_trsm_small // only float and double datatypes are supported as of now. if (dt != BLIS_DOUBLE && dt != BLIS_FLOAT) { - return BLIS_EXPECTED_REAL_DATATYPE; + return BLIS_EXPECTED_REAL_DATATYPE; } // A is expected to be triangular in trsm if (!bli_obj_is_upper_or_lower (a)) { - return BLIS_EXPECTED_TRIANGULAR_OBJECT; + return BLIS_EXPECTED_TRIANGULAR_OBJECT; } // can use other control structs - even can use array of function pointers, @@ -367,103 +373,98 @@ err_t bli_trsm_small // cases, can move the checks with more cases higher up. if (side == BLIS_LEFT) { - if (bli_obj_has_trans(a)) - { - if (dt == BLIS_DOUBLE) - { - if (bli_obj_is_upper(a)) - { - //A.'X = B; A is upper triangular; A has to be transposed; double precision + if (bli_obj_has_trans(a)) + { + if (dt == BLIS_DOUBLE) + { + if (bli_obj_is_upper(a)) + { + //A.'X = B; A is upper triangular; A has to be transposed; double precision #if 0 // planning to implement this in this iteration - return bli_dtrsm_small_AutXB(side, alpha, a, b, cntx, cntl); + return bli_dtrsm_small_AutXB(side, alpha, a, b, cntx, cntl); #else - return BLIS_NOT_YET_IMPLEMENTED; + return BLIS_NOT_YET_IMPLEMENTED; #endif - } - else - { - return BLIS_NOT_YET_IMPLEMENTED; - } - } - else if (dt == BLIS_FLOAT) - { - if (bli_obj_is_upper(a)) - { - //A.'X = B; A is upper triangular; A has to be transposed; single precision - return bli_strsm_small_AutXB(side, alpha, a, b, cntx, cntl); - //return BLIS_NOT_YET_IMPLEMENTED; - } - else - { - return BLIS_NOT_YET_IMPLEMENTED; - } - } - } - else - { - if (dt == BLIS_DOUBLE) - { - if (bli_obj_is_upper(a)) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - else - { - //AX = B; A is lower triangular; No transpose; double precision - return bli_dtrsm_small_AlXB(side, alpha, a, b, cntx, cntl); - //return BLIS_NOT_YET_IMPLEMENTED; - } - } - else if (dt == BLIS_FLOAT) - { - if (bli_obj_is_upper(a)) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - else - { - //AX = B; A is lower triangular; No transpose; single precision - return bli_strsm_small_AlXB(side, alpha, a, b, cntx, cntl); - //return BLIS_NOT_YET_IMPLEMENTED; - } - } - } + } + else + { + return BLIS_NOT_YET_IMPLEMENTED; + } + } + else if (dt == BLIS_FLOAT) + { + if (bli_obj_is_upper(a)) + { + //A.'X = B; A is upper triangular; A has to be transposed; single precision + return bli_strsm_small_AutXB(side, alpha, a, b, cntx, cntl); + } + else + { + return BLIS_NOT_YET_IMPLEMENTED; + } + } } else { - if (bli_obj_has_trans(a)) - { - if (dt == BLIS_DOUBLE) - { - if (bli_obj_is_upper(a)) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - else - { - //XA.' = B; A is lower triangular; A has to be transposed; double precision - return bli_dtrsm_small_XAltB(side, alpha, a, b, cntx, cntl); - //return BLIS_NOT_YET_IMPLEMENTED; - } - } - else if (dt == BLIS_FLOAT) - { - if (bli_obj_is_upper(a)) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - else - { - //XA.' = B; A is lower triangular; A has to be transposed; single precision - //return BLIS_NOT_YET_IMPLEMENTED; - return bli_strsm_small_XAltB(side, alpha, a, b, cntx, cntl); - } - } - } - else - { - return BLIS_NOT_YET_IMPLEMENTED; - } + if (dt == BLIS_DOUBLE) + { + if (bli_obj_is_upper(a)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //AX = B; A is lower triangular; No transpose; double precision + return bli_dtrsm_small_AlXB(side, alpha, a, b, cntx, cntl); + } + } + else if (dt == BLIS_FLOAT) + { + if (bli_obj_is_upper(a)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //AX = B; A is lower triangular; No transpose; single precision + return bli_strsm_small_AlXB(side, alpha, a, b, cntx, cntl); + } + } + } + } + else + { + if (bli_obj_has_trans(a)) + { + if (dt == BLIS_DOUBLE) + { + if (bli_obj_is_upper(a)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //XA.' = B; A is lower triangular; A has to be transposed; double precision + return bli_dtrsm_small_XAltB(side, alpha, a, b, cntx, cntl); + } + } + else if (dt == BLIS_FLOAT) + { + if (bli_obj_is_upper(a)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //XA.' = B; A is lower triangular; A has to be transposed; single precision + return bli_strsm_small_XAltB(side, alpha, a, b, cntx, cntl); + } + } + } + else + { + return BLIS_NOT_YET_IMPLEMENTED; + } } return BLIS_NOT_YET_IMPLEMENTED; @@ -471,13 +472,13 @@ err_t bli_trsm_small static void trsm_small_AlXB ( - float *A, - float *B, - int M, - int N, - int lda, - int ldb - ) + float *A, + float *B, + int M, + int N, + int lda, + int ldb + ) { int i; int j; @@ -490,14 +491,14 @@ static void trsm_small_AlXB ( float lkk_inv = 1.0/A[k+k*lda]; for (j = 0; j < N; j++) - { - B[k + j*ldb] *= lkk_inv; + { + B[k + j*ldb] *= lkk_inv; - for (i = k+1; i < M; i++) - { - B[i + j*ldb] -= A[i + k*lda] * B[k + j*ldb]; - } - } + for (i = k+1; i < M; i++) + { + B[i + j*ldb] -= A[i + k*lda] * B[k + j*ldb]; + } + } }// k -loop }// end of function @@ -505,46 +506,46 @@ static void trsm_small_AlXB ( // Test code: void gemm_small( float *ptr_l, - float *ptr_b, - int blk_m, - int blk_n, - float *ptr_gemmOut, - int cs_l, - int cs_b, - int rs_l, - int rs_b, - float alpha, - float beta) + float *ptr_b, + int blk_m, + int blk_n, + float *ptr_gemmOut, + int cs_l, + int cs_b, + int rs_l, + int rs_b, + float alpha, + float beta) { int i, j, k; for (i = 0; i < blk_m; i++) { for (j = 0; j < blk_n; j++) - { - float t = 0.0; - for (k = 0; k < blk_m; k++) - { - t += (ptr_l[i*rs_l + k* cs_l] * ptr_b[k*rs_b + j*cs_b]); - } - ptr_gemmOut[i*rs_b + j*cs_b] = beta * ptr_gemmOut[i*rs_b + j*cs_b] + alpha * t; - } + { + float t = 0.0; + for (k = 0; k < blk_m; k++) + { + t += (ptr_l[i*rs_l + k* cs_l] * ptr_b[k*rs_b + j*cs_b]); + } + ptr_gemmOut[i*rs_b + j*cs_b] = beta * ptr_gemmOut[i*rs_b + j*cs_b] + alpha * t; + } } } /* * AX = Alpha*B, Double precision, A:lower triangular -SUPPORTS MATRIX SIZE OF THE FORM 16X4*i, WHERE i IS AN INTEGER + * THIS KERNEL SUPPORTS MATRIX SIZE OF THE FORM BLI_AlXB_M_DPX4*i, WHERE i IS AN INTEGER */ static err_t bli_dtrsm_small_AlXB ( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) { obj_t alpha, beta; // gemm parameters obj_t Ga, Gb, Gc; // for GEMM @@ -566,7 +567,7 @@ static err_t bli_dtrsm_small_AlXB ( double *L = a->buffer; double *B = b->buffer; - if (m !=16 || (n&3) != 0) + if (m != BLI_AlXB_M_DP || (n&3) != 0) { return BLIS_NOT_YET_IMPLEMENTED; } @@ -592,7 +593,7 @@ static err_t bli_dtrsm_small_AlXB ( //first block of trsm Gb.buffer = (void*)(B + i); - if (alphaVal != 1) + if (alphaVal != 1) { if (isUnitDiag == 0) { @@ -655,6 +656,7 @@ static err_t bli_dtrsm_small_AlXB ( /* * AX = Alpha*B, Single precision, A: lower triangular + * This kernel implementation supports matrices A and B such that m is equal to BLI_AlXB_M_SP and n is mutiple of 8 */ static err_t bli_strsm_small_AlXB ( side_t side, @@ -663,7 +665,7 @@ static err_t bli_strsm_small_AlXB ( obj_t* b, cntx_t* cntx, cntl_t* cntl - ) + ) { obj_t alpha, beta; // gemm parameters obj_t Ga, Gb, Gc; // for GEMM @@ -685,13 +687,13 @@ static err_t bli_strsm_small_AlXB ( float *L = a->buffer; float *B = b->buffer; - if (m != 16 || (n%8) != 0) + if (m != BLI_AlXB_M_SP || (n&7) != 0) { - return BLIS_NOT_YET_IMPLEMENTED; + return BLIS_NOT_YET_IMPLEMENTED; } if ( (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) { - return BLIS_NOT_YET_IMPLEMENTED; + return BLIS_NOT_YET_IMPLEMENTED; } alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); @@ -719,30 +721,30 @@ static err_t bli_strsm_small_AlXB ( //trsm of first 8xn block if (alphaVal != 1) { - if (isUnitDiag == 0) - { - blis_strsm_microkernel_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - fp_blis_strsm_microkernel = blis_strsm_microkernel; - } - else - { - blis_strsm_microkernel_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; - } + if (isUnitDiag == 0) + { + blis_strsm_microkernel_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_strsm_microkernel = blis_strsm_microkernel; + } + else + { + blis_strsm_microkernel_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; + } bli_setsc( alphaVal, 0.0, &beta ); } else { - if (isUnitDiag == 0) - { - blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - fp_blis_strsm_microkernel = blis_strsm_microkernel; - } - else - { - blis_strsm_microkernel_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; - } + if (isUnitDiag == 0) + { + blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_strsm_microkernel = blis_strsm_microkernel; + } + else + { + blis_strsm_microkernel_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; + } } //gemm update @@ -757,17 +759,17 @@ static err_t bli_strsm_small_AlXB ( //trsm of remaining blocks for (i = blk_size; i < m; i += blk_size) { - Gb.buffer = (void*)(B + i); + Gb.buffer = (void*)(B + i); - fp_blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT - { - Ga.buffer = (void*)(L + j + i*lda); - Gc.buffer = (void*)(B + j); + for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + { + Ga.buffer = (void*)(L + j + i*lda); + Gc.buffer = (void*)(B + j); - bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb - } + bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + } } // End of for loop - i @@ -776,35 +778,37 @@ static err_t bli_strsm_small_AlXB ( void trsm_block_c(float *ptr_l, float *ptr_b, int blk_height, int blk_width, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) { - int i, j, k, l; - float inv_l; + int i, j, k, l; + float inv_l; - inv_l = 1.0 / *ptr_l; + inv_l = 1.0 / *ptr_l; - for (j = 0; j < numCols_b; j += blk_width) - { - for (l = j; l < (j+blk_width); l++) - { - ptr_b[l*cs_b] = ptr_b[l*cs_b] * inv_l; - } + for (j = 0; j < numCols_b; j += blk_width) + { + for (l = j; l < (j+blk_width); l++) + { + ptr_b[l*cs_b] = ptr_b[l*cs_b] * inv_l; + } - for (i = 1; i < blk_height; i++) - { - for (l = j; l < (j+blk_width); l++) - { - for (k = 0; k < i; k++) - { - ptr_b[i*rs_b + l*cs_b] -= (ptr_b[k*rs_b + l*cs_b] * ptr_l[i*rs_l + k*cs_l]); - } - ptr_b[i*rs_b + l*cs_b] = ptr_b[i*rs_b + l*cs_b] / ptr_l[i*rs_l + i*cs_l]; - } - } - } + for (i = 1; i < blk_height; i++) + { + for (l = j; l < (j+blk_width); l++) + { + for (k = 0; k < i; k++) + { + ptr_b[i*rs_b + l*cs_b] -= (ptr_b[k*rs_b + l*cs_b] * ptr_l[i*rs_l + k*cs_l]); + } + ptr_b[i*rs_b + l*cs_b] = ptr_b[i*rs_b + l*cs_b] / ptr_l[i*rs_l + i*cs_l]; + } + } + } } /* * XA' = Alpha*B, Double precision, A:lower triangular + * This kernel implementation supports matrices A and B such that + * m and n are multiples of 4 and n less than or equal to BLI_XAltB_N_DP */ static err_t bli_dtrsm_small_XAltB( @@ -814,7 +818,7 @@ static err_t bli_dtrsm_small_XAltB( obj_t* b, cntx_t* cntx, cntl_t* cntl - ) + ) { int m = bli_obj_length(a); // number of rows of matrix B @@ -833,36 +837,36 @@ static err_t bli_dtrsm_small_XAltB( double *L = a->buffer; double *B = b->buffer; - if ((m%4) != 0 || (n%4) != 0) + if ((m&3) != 0 || (n&3) != 0) { - return BLIS_NOT_YET_IMPLEMENTED; + return BLIS_NOT_YET_IMPLEMENTED; } - if ( n > 64 || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + if ( n > BLI_XAltB_N_DP || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) { - return BLIS_NOT_YET_IMPLEMENTED; + return BLIS_NOT_YET_IMPLEMENTED; } alphaVal = *((double *)AlphaObj->buffer); if (alphaVal != 1) { - if (isUnitDiag == 0) - { - dtrsm_XAtB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - } - else - { - dtrsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - } + if (isUnitDiag == 0) + { + dtrsm_XAtB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + else + { + dtrsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } } else { - if (isUnitDiag == 0) - { - dtrsm_XAtB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - } - else - { - dtrsm_XAtB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - } + if (isUnitDiag == 0) + { + dtrsm_XAtB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + else + { + dtrsm_XAtB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } } return BLIS_SUCCESS; @@ -871,6 +875,8 @@ static err_t bli_dtrsm_small_XAltB( /* * XA' = Alpha*B, Single precision, A: lower triangular + * This kernel implementation supports matrices A and B such that + * m and n are multiples of 8 and n is less than or equal to BLI_XAltB_N_SP */ static err_t bli_strsm_small_XAltB( side_t side, @@ -879,7 +885,7 @@ static err_t bli_strsm_small_XAltB( obj_t* b, cntx_t* cntx, cntl_t* cntl - ) + ) { int m = bli_obj_length(a); // number of rows of matrix B int n = bli_obj_length(b); // number of columns of matrix B @@ -897,44 +903,46 @@ static err_t bli_strsm_small_XAltB( float *L = a->buffer; float *B = b->buffer; - if ((m%8) != 0 || (n%8) != 0) + if ((m&7) != 0 || (n&7) != 0) { - return BLIS_NOT_YET_IMPLEMENTED; + return BLIS_NOT_YET_IMPLEMENTED; } - if ( n > 128 || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + if ( n > BLI_XAltB_N_SP || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) { - return BLIS_NOT_YET_IMPLEMENTED; + return BLIS_NOT_YET_IMPLEMENTED; } alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); if (alphaVal != 1) { - if (isUnitDiag == 0) - { - trsm_XAtB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - } - else - { - trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - } + if (isUnitDiag == 0) + { + trsm_XAtB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + else + { + trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } } else { - if (isUnitDiag == 0) - { - trsm_XAtB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - } - else - { - trsm_XAtB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - } + if (isUnitDiag == 0) + { + trsm_XAtB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + else + { + trsm_XAtB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } } return BLIS_SUCCESS; } /* * A'X = Alpha*B, Single precision, A: upper triangular + * This kernel implementation supports matrices A and B such that + * m and n are multiples of 8, m is less than or equal to BLI_AutXB_M_SP and n is less than or equal to BLI_AutXB_N_SP */ static err_t bli_strsm_small_AutXB( side_t side, @@ -943,7 +951,7 @@ static err_t bli_strsm_small_AutXB( obj_t* b, cntx_t* cntx, cntl_t* cntl - ) + ) { int m = bli_obj_width(a); // number of rows of matrix A (since At, so width is taken) int n = bli_obj_width(b); // number of columns of matrix B @@ -961,38 +969,38 @@ static err_t bli_strsm_small_AutXB( float *L = a->buffer; float *B = b->buffer; - if ((m%8) != 0 || (n%8) != 0) + if ((m&7) != 0 || (n&7) != 0) { - return BLIS_NOT_YET_IMPLEMENTED; + return BLIS_NOT_YET_IMPLEMENTED; } - if ( m > 64 || n > 128 || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + if ( m > BLI_AutXB_M_SP || n > BLI_AutXB_N_SP || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) { - return BLIS_NOT_YET_IMPLEMENTED; + return BLIS_NOT_YET_IMPLEMENTED; } alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); if (alphaVal != 1) { - if (isUnitDiag == 0) - { - trsm_AutXB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - } - else - { - trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - } + if (isUnitDiag == 0) + { + trsm_AutXB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + else + { + trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } } else { - if (isUnitDiag == 0) - { - trsm_AutXB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - } - else - { - trsm_AutXB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - } + if (isUnitDiag == 0) + { + trsm_AutXB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + else + { + trsm_AutXB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } } return BLIS_SUCCESS; } @@ -1001,75 +1009,173 @@ static err_t bli_strsm_small_AutXB( * ALPHA != 1; */ static void blis_dtrsm_microkernel_alpha(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - double alphaVal - ) + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ) { - double ones = 1.0; - int j; - int cs_b_offset[2]; - double *ptr_b_dup; + double ones = 1.0; + int j; + int cs_b_offset[2]; + double *ptr_b_dup; - __m256d mat_b_col[4]; - __m256d mat_b_rearr[4]; - __m256d mat_a_cols[4]; - __m256d mat_a_cols_rearr[10]; - __m256d mat_a_diag_inv[4]; - __m256d reciprocal_diags; - __m256d alphaReg; + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags; + __m256d alphaReg; - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; - reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); - alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); + reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); + alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); - //read first set of 4x4 block of B into registers - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - //4th col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - numCols_b -= 4; // blk_width = 4 + numCols_b -= 4; // blk_width = 4 - //compute reciprocals of L(i,i) and broadcast in registers - mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); - mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); - reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); + reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); + + for(j = 0;j < numCols_b; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + } + //Last block trsm processing - for(j = 0;j < numCols_b; j += 4) - { ptr_b_dup = ptr_b; /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ @@ -1077,31 +1183,31 @@ static void blis_dtrsm_microkernel_alpha(double *ptr_l, mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); - ////unpackhigh//// + ////unpackhigh//// mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); - //extract a00 - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); - //extract diag a11 from a + //extract diag a11 from a mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); @@ -1110,7 +1216,7 @@ static void blis_dtrsm_microkernel_alpha(double *ptr_l, mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); //extract diag a22 from a @@ -1121,7 +1227,7 @@ static void blis_dtrsm_microkernel_alpha(double *ptr_l, mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); //extract diag a33 from a @@ -1131,217 +1237,190 @@ static void blis_dtrsm_microkernel_alpha(double *ptr_l, //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - ////unpackhigh//// + ////unpackhigh//// mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - //Read next set of B columns - ptr_b += (cs_b+cs_b_offset[1]); - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - - //Store the computed B columns + //Store the computed B columns _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - } - //Last block trsm processing - - ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); - //extract a00 - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - } /* *AX=B A=LOWER TRIANGULAR, NO TRANSPOSE, UNITDIAGONAL *ALPHA != 1; */ static void blis_dtrsm_microkernel_alpha_unitDiag(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - double alphaVal - ) + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ) { - int j; - int cs_b_offset[2]; - double *ptr_b_dup; + int j; + int cs_b_offset[2]; + double *ptr_b_dup; - __m256d mat_b_col[4]; - __m256d mat_b_rearr[4]; - __m256d mat_a_cols[4]; - __m256d mat_a_cols_rearr[10]; - __m256d alphaReg; + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + __m256d alphaReg; - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; - alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - //4th col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + numCols_b -= 4; // blk_width = 4 - numCols_b -= 4; // blk_width = 4 + for(j = 0;j < numCols_b; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + } + //Last block trsm processing - for(j = 0;j < numCols_b; j += 4) - { ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ ////unpacklow//// mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); - ////unpackhigh//// + ////unpackhigh//// mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) @@ -1356,205 +1435,226 @@ static void blis_dtrsm_microkernel_alpha_unitDiag(double *ptr_l, //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - ////unpackhigh//// + ////unpackhigh//// mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - //Read next set of B columns - ptr_b += (cs_b+cs_b_offset[1]); - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - - //Store the computed B columns + //Store the computed B columns _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - } - //Last block trsm processing - - ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); - - ///unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - } /* *AX = B A= LOWERTRIANGULAR, NO TRANSPOSE, NON-UNITDIAGONAL *ALPHA = 1 */ static void blis_dtrsm_microkernel(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ) + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ) { - double ones = 1.0; - int j; - int cs_b_offset[2]; - double *ptr_b_dup; + double ones = 1.0; + int j; + int cs_b_offset[2]; + double *ptr_b_dup; - __m256d mat_b_col[4]; - __m256d mat_b_rearr[4]; - __m256d mat_a_cols[4]; - __m256d mat_a_cols_rearr[10]; - __m256d mat_a_diag_inv[4]; - __m256d reciprocal_diags; + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags; - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; - reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); + reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); - //row2 = (cs_l << 1); - //row4 = (cs_l << 2); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - //4th col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - numCols_b -= 4; // blk_width = 4 + numCols_b -= 4; // blk_width = 4 - //compute reciprocals of L(i,i) and broadcast in registers - mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); - mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); - reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); + reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); + + for(j = 0;j < numCols_b; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + } + //Last block trsm processing - for(j = 0;j < numCols_b; j += 4) - { ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ ////unpacklow//// mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - ////unpackhigh//// + ////unpackhigh//// mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - //extract a00 - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); - //extract diag a11 from a + //extract diag a11 from a mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); @@ -1563,7 +1663,7 @@ static void blis_dtrsm_microkernel(double *ptr_l, mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); //extract diag a22 from a @@ -1574,224 +1674,194 @@ static void blis_dtrsm_microkernel(double *ptr_l, mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - ////unpackhigh//// + ////unpackhigh//// mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - //Read next set of B columns - ptr_b += (cs_b+cs_b_offset[1]); - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - - //Store the computed B columns + //Store the computed B columns _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - - } - //Last block trsm processing - - ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - - //extract a00 - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - - - } /* *AX = B A=LOWER TRIANGULAR, NO TRANSPOSE, UNITDIAGONAL *ALPHA = 1 */ static void blis_dtrsm_microkernel_unitDiag(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ) + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ) { + //double ones = 1.0; + int j; + int cs_b_offset[2]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + numCols_b -= 4; // blk_width = 4 - //double ones = 1.0; - int j; - int cs_b_offset[2]; - double *ptr_b_dup; + for(j = 0;j < numCols_b; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - __m256d mat_b_col[4]; - __m256d mat_b_rearr[4]; - __m256d mat_a_cols[4]; - __m256d mat_a_cols_rearr[10]; + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); - //row2 = (cs_l << 1); - //row4 = (cs_l << 2); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); - - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //4th col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - numCols_b -= 4; // blk_width = 4 + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - for(j = 0;j < numCols_b; j += 4) - { + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + } + //Last block trsm processing + ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ ////unpacklow//// mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - ////unpackhigh//// + ////unpackhigh//// mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) @@ -1803,13306 +1873,13149 @@ static void blis_dtrsm_microkernel_unitDiag(double *ptr_l, mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - ////unpackhigh//// + ////unpackhigh//// mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + //rearrange high elements - //Read next set of B columns - ptr_b += (cs_b+cs_b_offset[1]); - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - //Store the computed B columns + //Store the computed B columns _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - - } - //Last block trsm processing - - ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange high elements - - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - - } ///////////////////////////// AX=B /////////////////////////////// static void blis_strsm_microkernel_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) { - float ones = 1.0; - int j; - int cs_b_offset[6]; - //int row2, row4, row6; - float *ptr_b_dup; + float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; - //70 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_cols[8]; - __m256 mat_a_cols_rearr[36]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags; - __m256 alphaReg; + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags; + __m256 alphaReg; - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); - alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); - //row2 = (cs_l << 1); - //row4 = (cs_l << 2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); - //row6 = row2 + row4; - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L - /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ - //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers - //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. - //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); - mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); - mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); - mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); - mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //4rth col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //5th col - ptr_l += cs_l; - mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //6th col - ptr_l += cs_l; - mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //7th col - ptr_l += cs_l; - mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //7th col - ptr_l += cs_l; - mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - numCols_b -= 8; // blk_width = 8 + numCols_b -= 8; // blk_width = 8 - //compute reciprocals of L(i,i) and broadcast in registers - mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); - mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); - mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); - //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); - //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); - //reciprocal of diagnol elements - reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + //reciprocal of diagnol elements + reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); - //Start loop for cols of B to be processed in size of blk_width - for (j = 0; j < numCols_b; j += 8) - { - ptr_b_dup = ptr_b; + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); #else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - //Read next set of B columns - ptr_b += (cs_b + cs_b_offset[5]); - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - //end loop of cols - } + //end loop of cols + } - //Last block trsm processing - ptr_b_dup = ptr_b; + //Last block trsm processing + ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); #else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - //end loop of cols + //end loop of cols } static void blis_strsm_microkernel_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) { - //float ones = 1.0; - int j; - int cs_b_offset[6]; - //int row2, row4, row6; - float *ptr_b_dup; + //float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; - //70 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_cols[8]; - __m256 mat_a_cols_rearr[36]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags; - __m256 alphaReg; + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags; + __m256 alphaReg; - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); - alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); - //row2 = (cs_l << 1); - //row4 = (cs_l << 2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); - //row6 = row2 + row4; - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L - /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ - //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers - //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. - //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); - mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); - mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); - mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); - mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //4rth col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //5th col - ptr_l += cs_l; - mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //6th col - ptr_l += cs_l; - mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //7th col - ptr_l += cs_l; - mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //8th col - //ptr_l += cs_l; - //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //8th col + //ptr_l += cs_l; + //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - numCols_b -= 8; // blk_width = 8 + numCols_b -= 8; // blk_width = 8 - //compute reciprocals of L(i,i) and broadcast in registers - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + //compute reciprocals of L(i,i) and broadcast in registers + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); - //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); - //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); - //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); - //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); - //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); - //reciprocal of diagnol elements - //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + //reciprocal of diagnol elements + //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); - //Start loop for cols of B to be processed in size of blk_width - for (j = 0; j < numCols_b; j += 8) - { - ptr_b_dup = ptr_b; + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //extract diag a00 from a - //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); - //extract diag a11 from a - //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - //extract diag a22 from a - //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - //extract diag a33 from a - //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - //extract diag a44 from a - //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - //extract diag a55 from a - //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - //extract diag a66 from a - //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - //extract diag a77 from a - //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); #else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - //Read next set of B columns - ptr_b += (cs_b + cs_b_offset[5]); - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - //end loop of cols - } + //end loop of cols + } - //Last block trsm processing - ptr_b_dup = ptr_b; + //Last block trsm processing + ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //extract diag a00 from a - //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); - //extract diag a11 from a - //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - //extract diag a22 from a - //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - //extract diag a33 from a - //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - //extract diag a44 from a - //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - //extract diag a55 from a - //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - //extract diag a66 from a - //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - //extract diag a77 from a - //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); #else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - //end loop of cols + //end loop of cols } static void blis_strsm_microkernel_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) { - //float ones = 1.0; - int j; - int cs_b_offset[6]; - //int row2, row4, row6; - float *ptr_b_dup; + //float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; - //70 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_cols[8]; - __m256 mat_a_cols_rearr[36]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags; + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags; - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); - //row2 = (cs_l << 1); - //row4 = (cs_l << 2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); - //row6 = row2 + row4; - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L - /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ - //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers - //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. - //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); - mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); - mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); - mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); - mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //4rth col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //5th col - ptr_l += cs_l; - mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //6th col - ptr_l += cs_l; - mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //7th col - ptr_l += cs_l; - mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //8th col - //ptr_l += cs_l; - //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //8th col + //ptr_l += cs_l; + //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - numCols_b -= 8; // blk_width = 8 + numCols_b -= 8; // blk_width = 8 - //compute reciprocals of L(i,i) and broadcast in registers - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + //compute reciprocals of L(i,i) and broadcast in registers + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); - //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); - //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); - //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); - //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); - //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); - //reciprocal of diagnol elements - //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + //reciprocal of diagnol elements + //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); - //Start loop for cols of B to be processed in size of blk_width - for (j = 0; j < numCols_b; j += 8) - { - ptr_b_dup = ptr_b; + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //extract diag a00 from a - //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - //extract diag a11 from a - //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - //extract diag a22 from a - //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - //extract diag a33 from a - //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - //extract diag a44 from a - //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - //extract diag a55 from a - //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - //extract diag a66 from a - //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - //extract diag a77 from a - //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); #else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - //Read next set of B columns - ptr_b += (cs_b + cs_b_offset[5]); - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - //end loop of cols - } + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols + } - //Last block trsm processing - ptr_b_dup = ptr_b; + //Last block trsm processing + ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //extract diag a00 from a - //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - //extract diag a11 from a - //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - //extract diag a22 from a - //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - //extract diag a33 from a - //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - //extract diag a44 from a - //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - //extract diag a55 from a - //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - //extract diag a66 from a - //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - //extract diag a77 from a - //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); #else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - //end loop of cols + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols } static void blis_strsm_microkernel(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) { - float ones = 1.0; - int j; - int cs_b_offset[6]; - //int row2, row4, row6; - float *ptr_b_dup; + float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; - //70 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_cols[8]; - __m256 mat_a_cols_rearr[36]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags; + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags; - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); - //row2 = (cs_l << 1); - //row4 = (cs_l << 2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); - //row6 = row2 + row4; - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L - /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ - //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers - //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. - //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); - mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); - mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); - mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); - mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //4rth col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //5th col - ptr_l += cs_l; - mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //6th col - ptr_l += cs_l; - mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //7th col - ptr_l += cs_l; - mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //7th col - ptr_l += cs_l; - mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - numCols_b -= 8; // blk_width = 8 + numCols_b -= 8; // blk_width = 8 - //compute reciprocals of L(i,i) and broadcast in registers - mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); - mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); - mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); - //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); - //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); - //reciprocal of diagnol elements - reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + //reciprocal of diagnol elements + reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); - //Start loop for cols of B to be processed in size of blk_width - for (j = 0; j < numCols_b; j += 8) - { - ptr_b_dup = ptr_b; + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); #else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - //Read next set of B columns - ptr_b += (cs_b + cs_b_offset[5]); - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - //end loop of cols - } + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols + } - //Last block trsm processing - ptr_b_dup = ptr_b; + //Last block trsm processing + ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); #else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - //end loop of cols + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols } ///////////////////////////////////// XA'=B functions //////////////////////////////// static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - double alpha - ) + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alpha + ) { - double ones = 1.0; - int i,i1,i2,i3,i4,j,k,l; - int cs_b_offset[3]; - int cs_l_offset[3]; - double *ptr_b_dup; - - __m256d mat_b_col[4]; - __m256d mat_b_rearr[16][4]; - __m256d mat_a_cols_rearr[4]; - __m256d mat_a_blk_elems[16]; - __m256d mat_a_diag_inv[4]; - __m256d reciprocal_diags[2]; - __m256d alphaReg; - reciprocal_diags[0] = _mm256_broadcast_sd((double const *)(&ones)); - alphaReg = _mm256_broadcast_sd((double const *)&alpha); - - // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - - //read diag elems of L 4x4 block - mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l); - mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[1]); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - - reciprocal_diags[1] = reciprocal_diags[0]; - - //pack first 4 diags together - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 - - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 - - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); - - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //Broadcast A21 to A31 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); - - //Broadcast A32 to register - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - - /***************** first set of 4 cols of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 4) - { - /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A - //read 4x4 block of B into registers - - mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - - mat_b_rearr[0][0] = _mm256_mul_pd(mat_b_rearr[0][0], alphaReg); - mat_b_rearr[1][0] = _mm256_mul_pd(mat_b_rearr[1][0], alphaReg); - mat_b_rearr[2][0] = _mm256_mul_pd(mat_b_rearr[2][0], alphaReg); - mat_b_rearr[3][0] = _mm256_mul_pd(mat_b_rearr[3][0], alphaReg); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_pd(mat_b_rearr[0][0], mat_a_diag_inv[0]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_pd(mat_b_rearr[1][0], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_pd(mat_b_rearr[2][0], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_pd(mat_b_rearr[3][0], mat_a_diag_inv[3]); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - - i += 4; - ptr_b_dup += 4; - - } - - /***************** first set of 4 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width} - - for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row - { - ptr_l += 4; - ptr_b_dup += cs_b_offset[2]; - i1 += cs_b_offset[2]; - //printf("i1 = i3 = %g\n",*(ptr_l+i1)); - //Read next 4x4 block of A to get diag elements - i3 += cs_l_offset[2]; - mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l + i3); - mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[1]); - - //pack 4 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 - - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 - - //reciprocal of diagnal elements of A :- 0,1,2,3 - reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); - - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 4) - { - i = i1 + k; - mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - - mat_b_rearr[i2][0] = _mm256_mul_pd(mat_b_rearr[i2][0], alphaReg); - mat_b_rearr[i2][1] = _mm256_mul_pd(mat_b_rearr[i2][1], alphaReg); - mat_b_rearr[i2][2] = _mm256_mul_pd(mat_b_rearr[i2][2], alphaReg); - mat_b_rearr[i2][3] = _mm256_mul_pd(mat_b_rearr[i2][3], alphaReg); - i2++; - } - - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 4) // move across m - { - - //Broadcast A4,0 to A7,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - - //Broadcast A41 to A71 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); - - //Broadcast A4,2 to A7,2 to registers - mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); - - //Broadcast A4,3 to A7,3 to registers - mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); - - i += cs_l_offset[2]; - - for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); - - - i4 = k >> 2; - - //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - - - //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - //end loop of cols - - } - i2 += cs_b_offset[2]; - - } - - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //Broadcast A21 to A31 to registers - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; + double ones = 1.0; + int i,i1,i2,i3,i4,j,k,l; + int cs_b_offset[3]; + int cs_l_offset[3]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[16][4]; + __m256d mat_a_cols_rearr[4]; + __m256d mat_a_blk_elems[16]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags[2]; + __m256d alphaReg; + reciprocal_diags[0] = _mm256_broadcast_sd((double const *)(&ones)); + alphaReg = _mm256_broadcast_sd((double const *)&alpha); + + // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + + //read diag elems of L 4x4 block + mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[1]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 4 diags together + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); + + //Broadcast A32 to register + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + /***************** first set of 4 cols of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 4) + { + /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A + //read 4x4 block of B into registers + + mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + mat_b_rearr[0][0] = _mm256_mul_pd(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_pd(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_pd(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_pd(mat_b_rearr[3][0], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_pd(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_pd(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_pd(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_pd(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + + i += 4; + ptr_b_dup += 4; + + } + + /***************** first set of 4 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width} + + for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row + { + ptr_l += 4; + ptr_b_dup += cs_b_offset[2]; + i1 += cs_b_offset[2]; + //printf("i1 = i3 = %g\n",*(ptr_l+i1)); + //Read next 4x4 block of A to get diag elements + i3 += cs_l_offset[2]; + mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[1]); + + //pack 4 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 + + //reciprocal of diagnal elements of A :- 0,1,2,3 + reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 4) + { + i = i1 + k; + mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_pd(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_pd(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_pd(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_pd(mat_b_rearr[i2][3], alphaReg); + i2++; + } + + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 4) // move across m + { + + //Broadcast A4,0 to A7,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + + //Broadcast A41 to A71 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); + + //Broadcast A4,2 to A7,2 to registers + mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); + + //Broadcast A4,3 to A7,3 to registers + mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); + + i += cs_l_offset[2]; + + for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); + + + i4 = k >> 2; + + //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + + + //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + //end loop of cols + + } + i2 += cs_b_offset[2]; + + } + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - k = 0; - for (i = 0; i < numCols_b; i+=4) - { + k = 0; + for (i = 0; i < numCols_b; i+=4) + { - /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A + /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[k][0] = _mm256_mul_pd(mat_b_rearr[k][0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_pd(mat_b_rearr[k][0], mat_a_diag_inv[0]); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[k][1] = _mm256_mul_pd(mat_b_rearr[k][1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_pd(mat_b_rearr[k][1], mat_a_diag_inv[1]); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[k][2] = _mm256_mul_pd(mat_b_rearr[k][2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_pd(mat_b_rearr[k][2], mat_a_diag_inv[2]); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[k][3] = _mm256_mul_pd(mat_b_rearr[k][3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_pd(mat_b_rearr[k][3], mat_a_diag_inv[3]); - //Store the computed B columns + //Store the computed B columns - _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - k++; - } - } + k++; + } + } } static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - double alpha - ) + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alpha + ) { - int i,i1,i2,i3,i4,j,k,l; - int cs_b_offset[3]; - int cs_l_offset[3]; - double *ptr_b_dup; + int i,i1,i2,i3,i4,j,k,l; + int cs_b_offset[3]; + int cs_l_offset[3]; + double *ptr_b_dup; - __m256d mat_b_col[4]; - __m256d mat_b_rearr[16][4]; - __m256d mat_a_blk_elems[16]; - __m256d alphaReg; - alphaReg = _mm256_broadcast_sd((double const *)&alpha); + __m256d mat_b_col[4]; + __m256d mat_b_rearr[16][4]; + __m256d mat_a_blk_elems[16]; + __m256d alphaReg; + alphaReg = _mm256_broadcast_sd((double const *)&alpha); - // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // + // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - //Broadcast A21 to A31 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); + //Broadcast A21 to A31 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); - //Broadcast A32 to register - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); + //Broadcast A32 to register + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); - /***************** first set of 4 cols of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 4) - { - /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A - //read 4x4 block of B into registers + /***************** first set of 4 cols of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 4) + { + /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A + //read 4x4 block of B into registers - mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[0][0] = _mm256_mul_pd(mat_b_rearr[0][0], alphaReg); - mat_b_rearr[1][0] = _mm256_mul_pd(mat_b_rearr[1][0], alphaReg); - mat_b_rearr[2][0] = _mm256_mul_pd(mat_b_rearr[2][0], alphaReg); - mat_b_rearr[3][0] = _mm256_mul_pd(mat_b_rearr[3][0], alphaReg); + mat_b_rearr[0][0] = _mm256_mul_pd(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_pd(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_pd(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_pd(mat_b_rearr[3][0], alphaReg); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[0][0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[0][0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[0][0], mat_b_rearr[3][0]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[0][0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[0][0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[0][0], mat_b_rearr[3][0]);//d = c - (a*b) - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[1][0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[1][0], mat_b_rearr[3][0]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[1][0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[1][0], mat_b_rearr[3][0]);//d = c - (a*b) - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_rearr[2][0], mat_b_rearr[3][0]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_rearr[2][0], mat_b_rearr[3][0]);//d = c - (a*b) - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_b_rearr[0][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_rearr[1][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3][0]); + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_b_rearr[0][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_rearr[1][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3][0]); - i += 4; - ptr_b_dup += 4; + i += 4; + ptr_b_dup += 4; - } + } - /***************** first set of 4 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width} + /***************** first set of 4 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width} - for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row - { - ptr_l += 4; - ptr_b_dup += cs_b_offset[2]; - i1 += cs_b_offset[2]; - i3 += cs_l_offset[2]; - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 4) - { - i = i1 + k; - mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row + { + ptr_l += 4; + ptr_b_dup += cs_b_offset[2]; + i1 += cs_b_offset[2]; + i3 += cs_l_offset[2]; + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 4) + { + i = i1 + k; + mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[i2][0] = _mm256_mul_pd(mat_b_rearr[i2][0], alphaReg); - mat_b_rearr[i2][1] = _mm256_mul_pd(mat_b_rearr[i2][1], alphaReg); - mat_b_rearr[i2][2] = _mm256_mul_pd(mat_b_rearr[i2][2], alphaReg); - mat_b_rearr[i2][3] = _mm256_mul_pd(mat_b_rearr[i2][3], alphaReg); - i2++; - } + mat_b_rearr[i2][0] = _mm256_mul_pd(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_pd(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_pd(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_pd(mat_b_rearr[i2][3], alphaReg); + i2++; + } - i = 0; - i2 = 0; - for (l = 0; l < j; l += 4) // move across m - { + i = 0; + i2 = 0; + for (l = 0; l < j; l += 4) // move across m + { - //Broadcast A4,0 to A7,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + //Broadcast A4,0 to A7,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - //Broadcast A41 to A71 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); + //Broadcast A41 to A71 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); - //Broadcast A4,2 to A7,2 to registers - mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); + //Broadcast A4,2 to A7,2 to registers + mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); - //Broadcast A4,3 to A7,3 to registers - mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); + //Broadcast A4,3 to A7,3 to registers + mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); - i += cs_l_offset[2]; + i += cs_l_offset[2]; - for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B + for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); - i4 = k >> 2; + i4 = k >> 2; - //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - //end loop of cols + //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + //end loop of cols - } - i2 += cs_b_offset[2]; + } + i2 += cs_b_offset[2]; - } + } - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; - //Broadcast A21 to A31 to registers - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; + //Broadcast A21 to A31 to registers + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; - //Broadcast A32 to A72 to registers - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; + //Broadcast A32 to A72 to registers + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; - k = 0; - for (i = 0; i < numCols_b; i+=4) - { + k = 0; + for (i = 0; i < numCols_b; i+=4) + { - /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A + /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - //Store the computed B columns + //Store the computed B columns - _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - k++; - } + k++; + } - } + } } static void dtrsm_XAtB_block_allSmallSizedMatrices_unitDiag(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ) + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ) { - int i,i1,i2,i3,i4,j,k,l; - int cs_b_offset[3]; - int cs_l_offset[3]; - double *ptr_b_dup; + int i,i1,i2,i3,i4,j,k,l; + int cs_b_offset[3]; + int cs_l_offset[3]; + double *ptr_b_dup; - __m256d mat_b_col[4]; - __m256d mat_b_rearr[16][4]; - __m256d mat_a_blk_elems[16]; + __m256d mat_b_col[4]; + __m256d mat_b_rearr[16][4]; + __m256d mat_a_blk_elems[16]; - // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // + // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - //Broadcast A21 to A31 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); + //Broadcast A21 to A31 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); - //Broadcast A32 to register - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); + //Broadcast A32 to register + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); - /***************** first set of 4 cols of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 4) - { - /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A - //read 4x4 block of B into registers + /***************** first set of 4 cols of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 4) + { + /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A + //read 4x4 block of B into registers - mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[0][0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[0][0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[0][0], mat_b_rearr[3][0]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[0][0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[0][0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[0][0], mat_b_rearr[3][0]);//d = c - (a*b) - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[1][0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[1][0], mat_b_rearr[3][0]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[1][0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[1][0], mat_b_rearr[3][0]);//d = c - (a*b) - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_rearr[2][0], mat_b_rearr[3][0]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_rearr[2][0], mat_b_rearr[3][0]);//d = c - (a*b) - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_b_rearr[0][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_rearr[1][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3][0]); + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_b_rearr[0][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_rearr[1][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3][0]); - i += 4; - ptr_b_dup += 4; + i += 4; + ptr_b_dup += 4; - } + } - /***************** first set of 4 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width} + /***************** first set of 4 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width} - for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row - { - ptr_l += 4; - ptr_b_dup += cs_b_offset[2]; - i1 += cs_b_offset[2]; - i3 += cs_l_offset[2]; - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 4) - { - i = i1 + k; - mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row + { + ptr_l += 4; + ptr_b_dup += cs_b_offset[2]; + i1 += cs_b_offset[2]; + i3 += cs_l_offset[2]; + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 4) + { + i = i1 + k; + mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - i2++; - } + i2++; + } - i = 0; - i2 = 0; - for (l = 0; l < j; l += 4) // move across m - { + i = 0; + i2 = 0; + for (l = 0; l < j; l += 4) // move across m + { - //Broadcast A4,0 to A7,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + //Broadcast A4,0 to A7,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - //Broadcast A41 to A71 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); + //Broadcast A41 to A71 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); - //Broadcast A4,2 to A7,2 to registers - mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); + //Broadcast A4,2 to A7,2 to registers + mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); - //Broadcast A4,3 to A7,3 to registers - mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); + //Broadcast A4,3 to A7,3 to registers + mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); - i += cs_l_offset[2]; + i += cs_l_offset[2]; - for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B + for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); - i4 = k >> 2; + i4 = k >> 2; - //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - //end loop of cols + //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + //end loop of cols - } - i2 += cs_b_offset[2]; + } + i2 += cs_b_offset[2]; - } + } - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; - //Broadcast A21 to A31 to registers - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; + //Broadcast A21 to A31 to registers + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; - //Broadcast A32 to A72 to registers - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; + //Broadcast A32 to A72 to registers + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; - k = 0; - for (i = 0; i < numCols_b; i+=4) - { + k = 0; + for (i = 0; i < numCols_b; i+=4) + { - /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A + /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - //Store the computed B columns + //Store the computed B columns - _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - k++; - } + k++; + } - } + } } static void dtrsm_XAtB_block_allSmallSizedMatrices(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ) + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ) { - double ones = 1.0; - int i,i1,i2,i3,i4,j,k,l; - int cs_b_offset[3]; - int cs_l_offset[3]; - double *ptr_b_dup; + double ones = 1.0; + int i,i1,i2,i3,i4,j,k,l; + int cs_b_offset[3]; + int cs_l_offset[3]; + double *ptr_b_dup; - __m256d mat_b_col[4]; - __m256d mat_b_rearr[16][4]; - __m256d mat_a_cols_rearr[4]; - __m256d mat_a_blk_elems[16]; - __m256d mat_a_diag_inv[4]; - __m256d reciprocal_diags[2]; - - reciprocal_diags[0] = _mm256_broadcast_sd((double const *)(&ones)); + __m256d mat_b_col[4]; + __m256d mat_b_rearr[16][4]; + __m256d mat_a_cols_rearr[4]; + __m256d mat_a_blk_elems[16]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_sd((double const *)(&ones)); - // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // + // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); - //read diag elems of L 4x4 block - mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l); - mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[1]); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - - reciprocal_diags[1] = reciprocal_diags[0]; - - //pack first 4 diags together - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 - - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 - - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); - - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //Broadcast A21 to A31 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); - - //Broadcast A32 to register - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - - /***************** first set of 4 cols of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 4) - { - /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A - //read 4x4 block of B into registers - - mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_pd(mat_b_rearr[0][0], mat_a_diag_inv[0]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_pd(mat_b_rearr[1][0], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_pd(mat_b_rearr[2][0], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_pd(mat_b_rearr[3][0], mat_a_diag_inv[3]); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - - i += 4; - ptr_b_dup += 4; - - } - - /***************** first set of 4 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width} - - for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row - { - ptr_l += 4; - ptr_b_dup += cs_b_offset[2]; - i1 += cs_b_offset[2]; - //printf("i1 = i3 = %g\n",*(ptr_l+i1)); - //Read next 4x4 block of A to get diag elements - i3 += cs_l_offset[2]; - mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l + i3); - mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[1]); - - //pack 4 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 - - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 - - //reciprocal of diagnal elements of A :- 0,1,2,3 - reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); - - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 4) - { - i = i1 + k; - mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - - i2++; - } - - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 4) // move across m - { - - //Broadcast A4,0 to A7,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - - //Broadcast A41 to A71 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); - - //Broadcast A4,2 to A7,2 to registers - mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); - - //Broadcast A4,3 to A7,3 to registers - mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); - - i += cs_l_offset[2]; - - for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); - - - i4 = k >> 2; - - //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - - - //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - //end loop of cols - - } - i2 += cs_b_offset[2]; - - } + //read diag elems of L 4x4 block + mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[1]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 4 diags together + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); + + //Broadcast A32 to register + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + /***************** first set of 4 cols of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 4) + { + /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A + //read 4x4 block of B into registers + + mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_pd(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_pd(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_pd(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_pd(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + + i += 4; + ptr_b_dup += 4; + + } + + /***************** first set of 4 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width} + + for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row + { + ptr_l += 4; + ptr_b_dup += cs_b_offset[2]; + i1 += cs_b_offset[2]; + //printf("i1 = i3 = %g\n",*(ptr_l+i1)); + //Read next 4x4 block of A to get diag elements + i3 += cs_l_offset[2]; + mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[1]); + + //pack 4 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 + + //reciprocal of diagnal elements of A :- 0,1,2,3 + reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 4) + { + i = i1 + k; + mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + i2++; + } + + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 4) // move across m + { + + //Broadcast A4,0 to A7,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + + //Broadcast A41 to A71 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); + + //Broadcast A4,2 to A7,2 to registers + mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); + + //Broadcast A4,3 to A7,3 to registers + mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); + + i += cs_l_offset[2]; + + for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); + + + i4 = k >> 2; + + //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + + + //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + //end loop of cols + + } + i2 += cs_b_offset[2]; + + } - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //Broadcast A21 to A31 to registers - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //Broadcast A32 to A72 to registers - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; + //Broadcast A32 to A72 to registers + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - k = 0; - for (i = 0; i < numCols_b; i+=4) - { + k = 0; + for (i = 0; i < numCols_b; i+=4) + { - /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A + /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[k][0] = _mm256_mul_pd(mat_b_rearr[k][0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_pd(mat_b_rearr[k][0], mat_a_diag_inv[0]); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[k][1] = _mm256_mul_pd(mat_b_rearr[k][1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_pd(mat_b_rearr[k][1], mat_a_diag_inv[1]); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[k][2] = _mm256_mul_pd(mat_b_rearr[k][2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_pd(mat_b_rearr[k][2], mat_a_diag_inv[2]); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[k][3] = _mm256_mul_pd(mat_b_rearr[k][3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_pd(mat_b_rearr[k][3], mat_a_diag_inv[3]); - //Store the computed B columns + //Store the computed B columns - _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - k++; - } + k++; + } - } + } } #if OPT_CACHE_BLOCKING_L1 //new intrinsic kernels static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) { - float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags[2]; + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; - reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); - //read diag elems of L 16x16 block - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); - reciprocal_diags[1] = reciprocal_diags[0]; + reciprocal_diags[1] = reciprocal_diags[0]; - //pack first 8 diags together - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); -#if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); -#endif - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; - //Read next 8x8 block of A to get diag elements - i3 += cs_l_offset[6]; - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); - //pack 8 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { #if GEMM_ACCUM_A - i = i1 + r; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); #endif - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - ptr_l_dup = ptr_l; - i4 = i2 + r; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - i4 = k >> 3; - ptr_l_dup += cs_l; + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #endif - //end loop of cols - } - i2 += cs_b_offset[6]; - i += cs_l_offset[6]; - } - //trsm solve + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - { - i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A #if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); #endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; #if GEMM_ACCUM_A - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); #else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); #endif #if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } - } - } //numRows of A - ///////////////////loop ends ///////////////////// + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// } static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) { - float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags[2]; - __m256 alphaReg; + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; - reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); - alphaReg = _mm256_broadcast_ss((float const *)&alpha); + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); - //read diag elems of L 16x16 block - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); - reciprocal_diags[1] = reciprocal_diags[0]; + reciprocal_diags[1] = reciprocal_diags[0]; - //pack first 8 diags together - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); #if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); #endif - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; - //Read next 8x8 block of A to get diag elements - i3 += cs_l_offset[6]; - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); - //pack 8 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { #if GEMM_ACCUM_A - i = i1 + r; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); #endif - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - ptr_l_dup = ptr_l; - i4 = i2 + r; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - i4 = k >> 3; - ptr_l_dup += cs_l; + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #endif - //end loop of cols - } - i2 += cs_b_offset[6]; - i += cs_l_offset[6]; - } - //trsm solve + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - { - i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A #if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); - - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); #endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; #if GEMM_ACCUM_A - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); #else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); #endif #if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - //Store the computed B columns + //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } - } - } //numRows of A - ///////////////////loop ends ///////////////////// + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// } static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) { - //float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags[2]; + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); -#if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + //(Row0) + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); -#endif + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - //(Row0) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + //////////////////////////////////////////////////////////////////////////////// - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } - //////////////////////////////////////////////////////////////////////////////// + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - i3 += cs_l_offset[6]; - - i = 0; - i2 = 0; - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { + i = 0; + i2 = 0; + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { #if GEMM_ACCUM_A - i = i1 + r; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); #endif - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - ptr_l_dup = ptr_l; - i4 = i2 + r; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - i4 = k >> 3; - ptr_l_dup += cs_l; + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #endif - //end loop of cols - } - i2 += cs_b_offset[6]; - i += cs_l_offset[6]; - } - //trsm solve + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - { - i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A #if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); #endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; #if GEMM_ACCUM_A - //(Row0): already done + //(Row0): already done #else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); #endif #if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } - } - } //numRows of A - ///////////////////loop ends ///////////////////// + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// } static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) { - //float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags[2]; - __m256 alphaReg; - alphaReg = _mm256_broadcast_ss((float const *)&alpha); + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); #if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); #endif - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); - - //(Row0) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0) + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - i3 += cs_l_offset[6]; + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; - i = 0; - i2 = 0; - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { + i = 0; + i2 = 0; + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { #if GEMM_ACCUM_A - i = i1 + r; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); #endif - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - ptr_l_dup = ptr_l; - i4 = i2 + r; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - i4 = k >> 3; - ptr_l_dup += cs_l; + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; #if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #endif - //end loop of cols - } - i2 += cs_b_offset[6]; - i += cs_l_offset[6]; - } - //trsm solve + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - { - i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A #if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); - - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); #endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; #if GEMM_ACCUM_A - //(Row0): already done + //(Row0): already done #else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); #endif #if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - //////////////////////////////////////////////////////////////////////////////// + + //////////////////////////////////////////////////////////////////////////////// - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } - } - } //numRows of A - ///////////////////loop ends ///////////////////// + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// } #else //rel 1.0 intrisic kernels (NOT OPT_CACHE_BLOCKING_L1) static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) { - float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[16][8]; - __m256 mat_a_cols_rearr[8]; - __m256 mat_a_blk_elems[64]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags[2]; - - reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - //read diag elems of L 16x16 block - mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); - mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); - mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); - mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); - mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); - mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - - reciprocal_diags[1] = reciprocal_diags[0]; - - //pack first 8 diags together - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - - //Read next 8x8 block of A to get diag elements - i3 += cs_l_offset[6]; - mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); - - //pack 8 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 8) - { - i = i1 + k; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - i2++; - } - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); - - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); - - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); - mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); - mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); - mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); - mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); - - // _mm256_permute2f128_ps() - - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); - mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); - mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); - mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); - mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); - mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); - mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); - mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); - - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); - mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); - mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); - mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); - mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); - mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); - mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); - mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); - - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); - mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); - mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); - mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); - mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); - mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); - mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); - mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); - - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); - mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); - mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); - mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); - mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); - mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); - mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); - mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); - - i += cs_l_offset[6]; - - - for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - i4 = k >> 3; - - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) - - //end loop of cols - } - i2 += cs_b_offset[6]; - } - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - k = 0; - for (i = 0; i < numCols_b; i+=8) - { - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - - _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } - - - } - ///////////////////loop ends ///////////////////// + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + __m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// } static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) { - float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[16][8]; - __m256 mat_a_cols_rearr[8]; - __m256 mat_a_blk_elems[64]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags[2]; - __m256 alphaReg; - - reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); - alphaReg = _mm256_broadcast_ss((float const *)&alpha); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - //read diag elems of L 16x16 block - mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); - mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); - mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); - mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); - mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); - mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - - reciprocal_diags[1] = reciprocal_diags[0]; - - //pack first 8 diags together - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); - mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); - mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); - mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); - mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); - mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); - mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); - mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - - //Read next 8x8 block of A to get diag elements - i3 += cs_l_offset[6]; - mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); - - //pack 8 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 8) - { - i = i1 + k; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); - mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); - mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); - mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); - mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); - mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); - mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); - mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); - - i2++; - } - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); - - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); - - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); - mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); - mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); - mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); - mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); - - // _mm256_permute2f128_ps() - - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); - mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); - mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); - mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); - mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); - mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); - mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); - mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); - - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); - mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); - mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); - mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); - mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); - mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); - mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); - mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); - - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); - mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); - mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); - mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); - mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); - mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); - mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); - mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); - - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); - mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); - mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); - mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); - mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); - mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); - mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); - mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); - - i += cs_l_offset[6]; - - - for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - i4 = k >> 3; - - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) - - //end loop of cols - } - i2 += cs_b_offset[6]; - } - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - k = 0; - for (i = 0; i < numCols_b; i+=8) - { - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - - _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); - k++; - } - - - } - ///////////////////loop ends ///////////////////// + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + __m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); + mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); + mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); + mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); + mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); + mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); + mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); + mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); + mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); + + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + k++; + } + + + } + ///////////////////loop ends ///////////////////// } static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) { - //float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup; + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[16][8]; - //__m256 mat_a_cols_rearr[8]; - __m256 mat_a_blk_elems[64]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags[2]; + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + //__m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - //(Row0) - mat_b_col[0] = mat_b_rearr[0][0]; + //(Row0) + mat_b_col[0] = mat_b_rearr[0][0]; - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - i3 += cs_l_offset[6]; + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 8) - { - i = i1 + k; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - i2++; - } - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); - - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); - - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); - mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); - mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); - mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); - mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); - - // _mm256_permute2f128_ps() - - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); - mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); - mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); - mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); - mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); - mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); - mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); - mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); - - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); - mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); - mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); - mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); - mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); - mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); - mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); - mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); - - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); - mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); - mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); - mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); - mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); - mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); - mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); - mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); - - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); - mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); - mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); - mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); - mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); - mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); - mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); - mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); - - i += cs_l_offset[6]; - - for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - i4 = k >> 3; - - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) - //end loop of cols - } - i2 += cs_b_offset[6]; - } - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - k = 0; - for (i = 0; i < numCols_b; i+=8) - { - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A - - //(Row0): already done + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): already done - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - //Store the computed B columns + //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } - } - ///////////////////loop ends ///////////////////// + } + ///////////////////loop ends ///////////////////// } static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) { - //float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup; + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[16][8]; - //__m256 mat_a_cols_rearr[8]; - __m256 mat_a_blk_elems[64]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags[2]; - __m256 alphaReg; - alphaReg = _mm256_broadcast_ss((float const *)&alpha); + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + //__m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); - mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); - mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); - mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); - mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); - mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); - mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); - mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); - - //(Row0) - mat_b_col[0] = mat_b_rearr[0][0]; + mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); + mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); + mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); + mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); + mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); + + //(Row0) + mat_b_col[0] = mat_b_rearr[0][0]; - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - i3 += cs_l_offset[6]; + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 8) - { - i = i1 + k; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); - mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); - mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); - mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); - mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); - mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); - mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); - mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); - - i2++; - } - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); - - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); - - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); - mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); - mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); - mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); - mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); - - // _mm256_permute2f128_ps() - - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); - mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); - mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); - mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); - mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); - mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); - mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); - mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); - - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); - mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); - mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); - mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); - mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); - mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); - mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); - mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); - - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); - mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); - mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); - mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); - mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); - mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); - mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); - mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); - - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); - mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); - mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); - mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); - mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); - mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); - mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); - mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); - - i += cs_l_offset[6]; - - for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); + mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); + mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); + mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); + mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); + + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - i4 = k >> 3; - - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) - //end loop of cols - } - i2 += cs_b_offset[6]; - } - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - k = 0; - for (i = 0; i < numCols_b; i+=8) - { - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A - - //(Row0): already done + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): already done - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - //Store the computed B columns + //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } - } - ///////////////////loop ends ///////////////////// + } + ///////////////////loop ends ///////////////////// } #endif //OPT_CACHE_BLOCKING_L1 //////////////////////////// AutX=B /////////////////////// static void trsm_AutXB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) { - float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags[2]; + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; - reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); - //read diag elems of L 16x16 block - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); - reciprocal_diags[1] = reciprocal_diags[0]; + reciprocal_diags[1] = reciprocal_diags[0]; - //pack first 8 diags together - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); #if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); #endif - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); - i += cs_b_offset[6]; - ptr_b_dup += cs_b_offset[6]; - //i += 8; - //ptr_b_dup += 8; - } + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += cs_l_offset[6]; + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; - //Read next 8x8 block of A to get diag elements - i3 += 8; - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + //Read next 8x8 block of A to get diag elements + i3 += 8; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); - //pack 8 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += 8; - i1 += 8; - i = i1; - i2 = 0; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { #if GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ #endif - //i = 0; - ptr_l_dup = ptr_l; - i4 = i2; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - //{ - /////////////////// Partial Lower 8x8 block trsm of B - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); - mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); #else - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); - mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); - mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); - mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); - mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i4 = k >> 3; - ptr_l_dup++; + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #endif - //end loop of cols - //} - //i2 += cs_b_offset[6]; - i4 += 8; - } - //trsm solve + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - //{ - //i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A #if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ #endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i += cs_l; + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; #if GEMM_ACCUM_A - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); #else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); #endif #if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - //} - i += cs_b_offset[6]; - i2 += cs_b_offset[6]; - } - } //numRows of A - ///////////////////loop ends ///////////////////// + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// } static void trsm_AutXB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) { - float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags[2]; - __m256 alphaReg; + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; - reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); - alphaReg = _mm256_broadcast_ss((float const *)&alpha); + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); - //read diag elems of L 16x16 block - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); - reciprocal_diags[1] = reciprocal_diags[0]; + reciprocal_diags[1] = reciprocal_diags[0]; - //pack first 8 diags together - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); #if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); #endif - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); - i += cs_b_offset[6]; - ptr_b_dup += cs_b_offset[6]; - //i += 8; - //ptr_b_dup += 8; - } + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += cs_l_offset[6]; + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; - //Read next 8x8 block of A to get diag elements - i3 += 8; - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + //Read next 8x8 block of A to get diag elements + i3 += 8; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); - //pack 8 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += 8; - i1 += 8; - i = i1; - i2 = 0; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { #if GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ - - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); #endif - - //i = 0; - ptr_l_dup = ptr_l; - i4 = i2; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - //{ - /////////////////// Partial Lower 8x8 block trsm of B - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); - mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); #else - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); - mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); - mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); - mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); - mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i4 = k >> 3; - ptr_l_dup++; + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #endif - //end loop of cols - //} - //i2 += cs_b_offset[6]; - i4 += 8; - } - //trsm solve + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - //{ - //i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A #if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ - - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); #endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i += cs_l; + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; #if GEMM_ACCUM_A - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); #else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); #endif #if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - //} - i += cs_b_offset[6]; - i2 += cs_b_offset[6]; - } - } //numRows of A - ///////////////////loop ends ///////////////////// + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// } static void trsm_AutXB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) { - //float ones = 1.0; - int i, i1, i2, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; + //float ones = 1.0; + int i, i1, i2, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags[2]; + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); #if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); #endif - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ - //(Row0) + //(Row0) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); - i += cs_b_offset[6]; - ptr_b_dup += cs_b_offset[6]; - //i += 8; - //ptr_b_dup += 8; - } + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += cs_l_offset[6]; + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += 8; - i1 += 8; - i = i1; - i2 = 0; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { #if GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ #endif - - //i = 0; - ptr_l_dup = ptr_l; - i4 = i2; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - //{ - /////////////////// Partial Lower 8x8 block trsm of B - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); - mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); #else - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); - mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); - mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); - mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); - mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i4 = k >> 3; - ptr_l_dup++; + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #endif - //end loop of cols - //} - //i2 += cs_b_offset[6]; - i4 += 8; - } - //trsm solve + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - //{ - //i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A #if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ #endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i += cs_l; + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; #if GEMM_ACCUM_A - //(Row0): already done + //(Row0): already done #else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); #endif #if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - //} - i += cs_b_offset[6]; - i2 += cs_b_offset[6]; - } - } //numRows of A - ///////////////////loop ends ///////////////////// + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// } static void trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) { - //float ones = 1.0; - int i, i1, i2, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; + //float ones = 1.0; + int i, i1, i2, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags[2]; - __m256 alphaReg; - alphaReg = _mm256_broadcast_ss((float const *)&alpha); + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); #if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); #endif - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); - //(Row0) + //(Row0) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); - i += cs_b_offset[6]; - ptr_b_dup += cs_b_offset[6]; - //i += 8; - //ptr_b_dup += 8; - } + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += cs_l_offset[6]; + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += 8; - i1 += 8; - i = i1; - i2 = 0; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { #if GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ - - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); #endif - - //i = 0; - ptr_l_dup = ptr_l; - i4 = i2; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - //{ - /////////////////// Partial Lower 8x8 block trsm of B - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); - mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); #else - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); - mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); - mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); - mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); - mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i4 = k >> 3; - ptr_l_dup++; + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; #if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) #endif - //end loop of cols - //} - //i2 += cs_b_offset[6]; - i4 += 8; - } - //trsm solve + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - //{ - //i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A #if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ - - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); #endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i += cs_l; + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; #if GEMM_ACCUM_A - //(Row0): already done + //(Row0): already done #else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); #endif #if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) #endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); - //i += cs_l; + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange low elements + //Rearrange low elements #if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); #else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); #endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - //Rearrange high elements + //Rearrange high elements #if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); #else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); #endif - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - //} - i += cs_b_offset[6]; - i2 += cs_b_offset[6]; - } - } //numRows of A - ///////////////////loop ends ///////////////////// + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// } #endif From f0c3ef359f7c6c1687fb2671cb35deb346e00597 Mon Sep 17 00:00:00 2001 From: Kiran V Date: Thu, 4 Oct 2018 16:32:21 +0530 Subject: [PATCH 13/53] This is a fix to floating-point exception error for BLIS SGEMM with larger matrix sizes. BUG No: CPUPL-197 fixed by Thangaraj Santanu The bli_clock_min_diff() function in BLIS assumed that if the time taken is greater than 1 hour then the reading must be wrong. However this is not the case in general, while the other checks such as time taken closer to zero or nsec is ofcourse valid. gerrit review: http://git.amd.com:8080/#/c/118694/1/frame/base/bli_clock.c Change-Id: I9dc313d7c5fdc20684f67a516bf3237de3e0694a --- frame/base/bli_clock.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frame/base/bli_clock.c b/frame/base/bli_clock.c index 6f92d907b..216367975 100644 --- a/frame/base/bli_clock.c +++ b/frame/base/bli_clock.c @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (c) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -59,7 +60,9 @@ double bli_clock_min_diff( double time_min, double time_start ) // - under a nanosecond // is actually garbled due to the clocks being taken too closely together. if ( time_min <= 0.0 ) time_min = time_min_prev; - else if ( time_min > 3600.0 ) time_min = time_min_prev; + // To genuinely measure time for an application taking more than an hour, the below + // line is commented. If wrongly measuring higher time we could always use previous_min. + /* else if ( time_min > 3600.0 ) time_min = time_min_prev; */ else if ( time_min < 1.0e-9 ) time_min = time_min_prev; return time_min; From 78a6935483409ae277c766406e175772e820b1de Mon Sep 17 00:00:00 2001 From: sraut Date: Thu, 11 Oct 2018 10:49:40 +0530 Subject: [PATCH 14/53] Added comments for the change in syrk small matrix change. Change-Id: I958939e9953323730da49ef07d1b10e578837d82 --- config/zen/bli_family_zen.h | 2 ++ frame/3/syrk/bli_syrk_front.c | 27 ++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/config/zen/bli_family_zen.h b/config/zen/bli_family_zen.h index f61ec96d0..743c38f32 100644 --- a/config/zen/bli_family_zen.h +++ b/config/zen/bli_family_zen.h @@ -53,6 +53,8 @@ #define BLIS_SMALL_MATRIX_THRES_TRSM 32768 //128(128+128) => m*(m+n) #define BLIS_SMALL_MATRIX_A_THRES_TRSM 128 +#define BLIS_SMALL_MATRIX_A_THRES_M_SYRK 96 +#define BLIS_SMALL_MATRIX_A_THRES_N_SYRK 128 //#endif diff --git a/frame/3/syrk/bli_syrk_front.c b/frame/3/syrk/bli_syrk_front.c index fc50fcdc6..beedc3e6c 100644 --- a/frame/3/syrk/bli_syrk_front.c +++ b/frame/3/syrk/bli_syrk_front.c @@ -46,7 +46,9 @@ void bli_syrk_front ) { bli_init_once(); - +#ifdef BLIS_ENABLE_SMALL_MATRIX + gint_t status = BLIS_FAILURE; +#endif obj_t a_local; obj_t at_local; obj_t c_local; @@ -68,6 +70,29 @@ void bli_syrk_front bli_obj_set_as_root( &c_local ); // For syrk, the right-hand "B" operand is simply A^T. +#ifdef BLIS_ENABLE_SMALL_MATRIX + bli_obj_alias_to( a, &at_local ); + if (bli_obj_has_trans(a) != 0) + {//At*A operation + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &at_local ); + //call small gemm to perform syrk. + //gemm small matrix threshold check is done inside bli_gemm_small() which is good enough for syrk small matrix also. + status = bli_gemm_small( alpha, &a_local, &at_local, beta, &c_local, cntx, cntl ); + } + else if ((a->dim[0] <= BLIS_SMALL_MATRIX_A_THRES_M_SYRK && a->dim[1] < BLIS_SMALL_MATRIX_A_THRES_N_SYRK) || + (a->dim[0] < BLIS_SMALL_MATRIX_A_THRES_M_SYRK && a->dim[1] <= BLIS_SMALL_MATRIX_A_THRES_N_SYRK)) + {//A*At operation + bli_obj_set_conjtrans( BLIS_TRANSPOSE, &at_local ); + //call small gemm to perform syrk. + //Explicit matrix dimension threshold check in this else if section before calling bli_gemm_small() for syrk small matrix also. + status = bli_gemm_small( alpha, &a_local, &at_local, beta, &c_local, cntx, cntl ); + } + if ( status == BLIS_SUCCESS ) + { + return; + } +#endif + bli_obj_alias_to( a, &at_local ); bli_obj_induce_trans( &at_local ); From 0ae9585da1e3db1cf8034d4b16305a5883beb0d3 Mon Sep 17 00:00:00 2001 From: pradeeptrgit Date: Tue, 23 Oct 2018 09:36:23 +0530 Subject: [PATCH 15/53] Update version number to 1.2 Change-Id: Ibb31f6683cdecca6b218bc2f0c14701d7e92ebf3 --- so_version | 2 +- version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/so_version b/so_version index 7eb1c7917..15d6c7665 100644 --- a/so_version +++ b/so_version @@ -1,2 +1,2 @@ 1 -0.0 +2.0 diff --git a/version b/version index 267577d47..5625e59da 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.4.1 +1.2 From 7bf901e9265a1acd78e44c06f7178c8152c7e267 Mon Sep 17 00:00:00 2001 From: sraut Date: Tue, 18 Dec 2018 14:39:16 +0530 Subject: [PATCH 16/53] Fix on EPYC machine for multi instance performance issue, Issue: For the default values of mc, kc and nc with multi instance mode the performance across the cores dip drastically. Fix: After experimentation found different set of values (mc, kc and nc) which fits in the cache size, and performance across the remains same across all the cores. Change-Id: I98265e3b7e61cd7602a0cc5596240e86c08c03fe --- config/zen/bli_cntx_init_zen.c | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/config/zen/bli_cntx_init_zen.c b/config/zen/bli_cntx_init_zen.c index 6507f421b..0f3bfe680 100644 --- a/config/zen/bli_cntx_init_zen.c +++ b/config/zen/bli_cntx_init_zen.c @@ -111,15 +111,36 @@ void bli_cntx_init_zen( cntx_t* cntx ) // s d c z bli_blksz_init_easy( &blkszs[ BLIS_MR ], 6, 6, 3, 3 ); bli_blksz_init_easy( &blkszs[ BLIS_NR ], 16, 8, 8, 4 ); + +/* + Multi Instance performance degradation on different cores + a) CPU freq 2.6 Ghz + DDR4 2400 + Multi instance mode + mc = 240, kc = 512, and nc = 2040 + + b) CPU freq 2.4Ghz + DDR4 2400 + Multi Instance mode + either + mc = 240, kc = 512 and nc = 2040 + (or) + mc = 390, kc = 512 and nc = 4080 + + c) Higher frequency(3.1Ghz), single instance mode choose default value + mc = 510, kc = 1024 and nc = 4080 + +*/ + #ifdef BLIS_ENABLE_ZEN_BLOCK_SIZES // Zen optmized level 3 cache block sizes - bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 510, 144, 72 ); - bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 1024, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 240, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); #else bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); #endif - bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); bli_blksz_init_easy( &blkszs[ BLIS_AF ], 8, 8, -1, -1 ); bli_blksz_init_easy( &blkszs[ BLIS_DF ], 8, 8, -1, -1 ); From 6d267375c3a0543f20604d74cc678ad91db3b6f1 Mon Sep 17 00:00:00 2001 From: sraut Date: Wed, 19 Dec 2018 14:22:21 +0530 Subject: [PATCH 17/53] This commit improves the performance of multi-instance DGEMM when these multiple threads are binded to a CCX. Multi-Instance: Each thread runs a sequential DGEMM. Change-Id: I306920c8061b6dad61efac1dae68727f4ac27df6 --- config/zen/bli_cntx_init_zen.c | 40 ++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/config/zen/bli_cntx_init_zen.c b/config/zen/bli_cntx_init_zen.c index 0f3bfe680..db9e91f03 100644 --- a/config/zen/bli_cntx_init_zen.c +++ b/config/zen/bli_cntx_init_zen.c @@ -113,34 +113,60 @@ void bli_cntx_init_zen( cntx_t* cntx ) bli_blksz_init_easy( &blkszs[ BLIS_NR ], 16, 8, 8, 4 ); /* - Multi Instance performance degradation on different cores + Multi Instance performance improvement of DGEMM when binded to a CCX + In Multi instance each thread runs a sequential DGEMM. + a) CPU freq 2.6 Ghz - DDR4 2400 + DDR4 clock frequency 2400Mhz Multi instance mode mc = 240, kc = 512, and nc = 2040 - b) CPU freq 2.4Ghz - DDR4 2400 + b) CPU freq 2.2Ghz + DDR4 clock frequency 2400Mhz Multi Instance mode either mc = 240, kc = 512 and nc = 2040 (or) mc = 390, kc = 512 and nc = 4080 - c) Higher frequency(3.1Ghz), single instance mode choose default value + c) If BLIS is run in Single Instance mode choose mc = 510, kc = 1024 and nc = 4080 */ #ifdef BLIS_ENABLE_ZEN_BLOCK_SIZES // Zen optmized level 3 cache block sizes - bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 240, 144, 72 ); + /************************************************************************ + Below block sizes of DGEMM, works better in a multi instance mode, + for clock frequency of 2.6Ghz and DDR4 clock frequency of 2400Mhz + ************************************************************************/ + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 240, 144, 72 ); bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); + + /*********************************************************************************** + Below block sizes of DGEMM, gives better performance in a multi instance mode, + for clock frequency of 2.2Ghz and DDR4 clock frequency of 2400Mhz + **************************************************************************************/ + //bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 390, 144, 72 ); + //bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); + //bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); + + /****************************************************************************** + BLIS on single instance mode, gives better perfomance with + below mentioned default block size values + ********************************************************************************/ + // bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 510, 144, 72 ); + //bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 1024, 256, 256 ); + //bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); + #else + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); #endif - bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); + //bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); bli_blksz_init_easy( &blkszs[ BLIS_AF ], 8, 8, -1, -1 ); bli_blksz_init_easy( &blkszs[ BLIS_DF ], 8, 8, -1, -1 ); From 1f4eeee5175a8fc9ac312847c796ce6db5fe75b9 Mon Sep 17 00:00:00 2001 From: sraut Date: Wed, 19 Dec 2018 21:21:10 +0530 Subject: [PATCH 18/53] Fixed BLAS test failures of small matrix SYRK for single and double precision. Details: - SYRK for small matrix was implemented by reusing small GEMM routine. This was resulting in output written to the full C matrix, and C being symmetric the lower and upper triangles of C matrix contained same results. BLAS SYRK API spec demands either lower or upper triangle of C matrix to be written with results. So, this was resulting in BLAS test failures, even though testsuite of BLIS was passing small SYRK operation. - To fix BLAS test failures of small matrix SYRK, separate kernel routines are implemented for small SYRK for both single and double precision. The newly added small SYRK routines are in file kernels/zen/3/bli_syrk_small.c. Now the intermediate results of matrix C are written to a scratch buffer. Final results are written from scratch buffer to matrix C using SIMD copy to either lower or upper traingle part of matrix C. - Source and header files frame/3/syrk/bli_syrk_front.c and frame/3/syrk/bli_syrk_front.h are changed to invoke new small SYRK routines. Change-Id: I9cfb1116c93d150aefac673fca033952ecac97cb --- config/zen/bli_cntx_init_zen.c | 31 +- frame/3/syrk/bli_syrk_front.c | 41 +- frame/3/syrk/bli_syrk_front.h | 11 + kernels/zen/3/bli_syrk_small.c | 4186 ++++++++++++++++++++++++++++++++ 4 files changed, 4234 insertions(+), 35 deletions(-) create mode 100644 kernels/zen/3/bli_syrk_small.c diff --git a/config/zen/bli_cntx_init_zen.c b/config/zen/bli_cntx_init_zen.c index db9e91f03..be0eae894 100644 --- a/config/zen/bli_cntx_init_zen.c +++ b/config/zen/bli_cntx_init_zen.c @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -138,27 +139,27 @@ void bli_cntx_init_zen( cntx_t* cntx ) // Zen optmized level 3 cache block sizes /************************************************************************ Below block sizes of DGEMM, works better in a multi instance mode, - for clock frequency of 2.6Ghz and DDR4 clock frequency of 2400Mhz + for clock frequency of 2.2GHz and DDR4 clock frequency of 2400MHz ************************************************************************/ bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 240, 144, 72 ); bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); - /*********************************************************************************** - Below block sizes of DGEMM, gives better performance in a multi instance mode, - for clock frequency of 2.2Ghz and DDR4 clock frequency of 2400Mhz - **************************************************************************************/ - //bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 390, 144, 72 ); - //bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); - //bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); + /*********************************************************************************** + Below block sizes of DGEMM, gives better performance in a multi instance mode, + for clock frequency of 2.6GHz and DDR4 clock frequency of 2400MHz + **************************************************************************************/ + //bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 390, 144, 72 ); + //bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); + //bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); - /****************************************************************************** - BLIS on single instance mode, gives better perfomance with - below mentioned default block size values - ********************************************************************************/ - // bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 510, 144, 72 ); - //bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 1024, 256, 256 ); - //bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); + /****************************************************************************** + BLIS on single instance mode, gives better perfomance with + below mentioned default block size values + ********************************************************************************/ + //bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 510, 144, 72 ); + //bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 1024, 256, 256 ); + //bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); #else diff --git a/frame/3/syrk/bli_syrk_front.c b/frame/3/syrk/bli_syrk_front.c index beedc3e6c..219bf85e0 100644 --- a/frame/3/syrk/bli_syrk_front.c +++ b/frame/3/syrk/bli_syrk_front.c @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -71,26 +72,26 @@ void bli_syrk_front // For syrk, the right-hand "B" operand is simply A^T. #ifdef BLIS_ENABLE_SMALL_MATRIX - bli_obj_alias_to( a, &at_local ); - if (bli_obj_has_trans(a) != 0) - {//At*A operation - bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &at_local ); - //call small gemm to perform syrk. - //gemm small matrix threshold check is done inside bli_gemm_small() which is good enough for syrk small matrix also. - status = bli_gemm_small( alpha, &a_local, &at_local, beta, &c_local, cntx, cntl ); - } - else if ((a->dim[0] <= BLIS_SMALL_MATRIX_A_THRES_M_SYRK && a->dim[1] < BLIS_SMALL_MATRIX_A_THRES_N_SYRK) || - (a->dim[0] < BLIS_SMALL_MATRIX_A_THRES_M_SYRK && a->dim[1] <= BLIS_SMALL_MATRIX_A_THRES_N_SYRK)) - {//A*At operation - bli_obj_set_conjtrans( BLIS_TRANSPOSE, &at_local ); - //call small gemm to perform syrk. - //Explicit matrix dimension threshold check in this else if section before calling bli_gemm_small() for syrk small matrix also. - status = bli_gemm_small( alpha, &a_local, &at_local, beta, &c_local, cntx, cntl ); - } - if ( status == BLIS_SUCCESS ) - { - return; - } + bli_obj_alias_to( a, &at_local ); + if (bli_obj_has_trans(a) != 0) + {//At*A operation + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &at_local ); + //call small syrk. + //syrk small matrix threshold check is done inside bli_syrk_small(). + status = bli_syrk_small( alpha, &a_local, &at_local, beta, &c_local, cntx, cntl ); + } + else if ((a->dim[0] <= BLIS_SMALL_MATRIX_A_THRES_M_SYRK && a->dim[1] < BLIS_SMALL_MATRIX_A_THRES_N_SYRK) || + (a->dim[0] < BLIS_SMALL_MATRIX_A_THRES_M_SYRK && a->dim[1] <= BLIS_SMALL_MATRIX_A_THRES_N_SYRK)) + {//A*At operation + bli_obj_set_conjtrans( BLIS_TRANSPOSE, &at_local ); + //call small syrk. + //Explicit matrix dimension threshold check in this else if section before calling bli_syrk_small(). + status = bli_syrk_small( alpha, &a_local, &at_local, beta, &c_local, cntx, cntl ); + } + if ( status == BLIS_SUCCESS ) + { + return; + } #endif bli_obj_alias_to( a, &at_local ); diff --git a/frame/3/syrk/bli_syrk_front.h b/frame/3/syrk/bli_syrk_front.h index 83199eee1..bf8c23cde 100644 --- a/frame/3/syrk/bli_syrk_front.h +++ b/frame/3/syrk/bli_syrk_front.h @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -42,3 +43,13 @@ void bli_syrk_front rntm_t* rntm, cntl_t* cntl ); +err_t bli_syrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); \ No newline at end of file diff --git a/kernels/zen/3/bli_syrk_small.c b/kernels/zen/3/bli_syrk_small.c new file mode 100644 index 000000000..d10114b40 --- /dev/null +++ b/kernels/zen/3/bli_syrk_small.c @@ -0,0 +1,4186 @@ +/* + +BLIS +An object-based framework for developing high-performance BLAS-like +libraries. + +Copyright (C) 2018, Advanced Micro Devices, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of The University of Texas at Austin nor the names +of its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "immintrin.h" +#include "xmmintrin.h" +#include "blis.h" + +#ifdef BLIS_ENABLE_SMALL_MATRIX + +#define MR 32 +#define D_MR (MR >> 1) +#define NR 3 + +#define BLIS_ENABLE_PREFETCH +#define F_SCRATCH_DIM (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES) +static float A_pack[F_SCRATCH_DIM] __attribute__((aligned(64))); +static float C_pack[F_SCRATCH_DIM] __attribute__((aligned(64))); +#define D_BLIS_SMALL_MATRIX_THRES (BLIS_SMALL_MATRIX_THRES / 2 ) +#define D_BLIS_SMALL_M_RECT_MATRIX_THRES (BLIS_SMALL_M_RECT_MATRIX_THRES / 2) +#define D_BLIS_SMALL_K_RECT_MATRIX_THRES (BLIS_SMALL_K_RECT_MATRIX_THRES / 2) +#define D_SCRATCH_DIM (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES) +static double D_A_pack[D_SCRATCH_DIM] __attribute__((aligned(64))); +static double D_C_pack[D_SCRATCH_DIM] __attribute__((aligned(64))); +#define BLIS_ATBN_M_THRES 40 // Threshold value of M for/below which small matrix code is called. +#define AT_MR 4 // The kernel dimension of the A transpose SYRK kernel.(AT_MR * NR). +static err_t bli_ssyrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); + +static err_t bli_dsyrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); + +static err_t bli_ssyrk_small_atbn + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); + +static err_t bli_dsyrk_small_atbn + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); +/* +* The bli_syrk_small function will use the +* custom MRxNR kernels, to perform the computation. +* The custom kernels are used if the [M * N] < 240 * 240 +*/ +err_t bli_syrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ +#ifdef BLIS_ENABLE_MULTITHREADING + return BLIS_NOT_YET_IMPLEMENTED; +#endif + // If alpha is zero, scale by beta and return. + if (bli_obj_equals(alpha, &BLIS_ZERO)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + // if row major format return. + if ((bli_obj_row_stride( a ) != 1) || + (bli_obj_row_stride( b ) != 1) || + (bli_obj_row_stride( c ) != 1)) + { + return BLIS_INVALID_ROW_STRIDE; + } + + num_t dt = ((*c).info & (0x7 << 0)); + + if (bli_obj_has_trans( a )) + { + if (bli_obj_has_notrans( b )) + { + if (dt == BLIS_FLOAT) + { + return bli_ssyrk_small_atbn(alpha, a, b, beta, c, cntx, cntl); + } + else if (dt == BLIS_DOUBLE) + { + return bli_dsyrk_small_atbn(alpha, a, b, beta, c, cntx, cntl); + } + } + + return BLIS_NOT_YET_IMPLEMENTED; + } + + if (dt == BLIS_DOUBLE) + { + return bli_dsyrk_small(alpha, a, b, beta, c, cntx, cntl); + } + + if (dt == BLIS_FLOAT) + { + return bli_ssyrk_small(alpha, a, b, beta, c, cntx, cntl); + } + + return BLIS_NOT_YET_IMPLEMENTED; +}; + + +static err_t bli_ssyrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + int M = bli_obj_length( c ); // number of rows of Matrix C + int N = bli_obj_width( c ); // number of columns of Matrix C + int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + int L = M * N; + + if ((((L) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES)) + || ((M < BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) + { + + int lda = bli_obj_col_stride(a); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + int ldb = bli_obj_col_stride(b); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + int ldc_matC = bli_obj_col_stride( c ); // column stride of matrix C + int ldc = M;//bli_obj_col_stride( c ); // column stride of static buffer for matrix C + int row_idx, col_idx, k; + int rs_matC = bli_obj_row_stride( c ); + int rsc = 1; + float *A = a->buffer; // pointer to elements of Matrix A + float *B = b->buffer; // pointer to elements of Matrix B + float *C = C_pack; // pointer to elements of Matrix C + float *matCbuf = c->buffer; + + float *tA = A, *tB = B, *tC = C;//, *tA_pack; + float *tA_packed; // temprorary pointer to hold packed A memory pointer + int row_idx_packed; //packed A memory row index + int lda_packed; //lda of packed A + int col_idx_start; //starting index after A matrix is packed. + dim_t tb_inc_row = 1; // row stride of matrix B + dim_t tb_inc_col = ldb; // column stride of matrix B + __m256 ymm4, ymm5, ymm6, ymm7; + __m256 ymm8, ymm9, ymm10, ymm11; + __m256 ymm12, ymm13, ymm14, ymm15; + __m256 ymm0, ymm1, ymm2, ymm3; + + int n_remainder; // If the N is non multiple of 3.(N%3) + int m_remainder; // If the M is non multiple of 32.(M%32) + + float *alpha_cast, *beta_cast; // alpha, beta multiples + alpha_cast = (alpha->buffer); + beta_cast = (beta->buffer); + int required_packing_A = 1; + + // when N is equal to 1 call GEMV instead of SYRK + if (N == 1) + { + bli_gemv + ( + alpha, + a, + b, + beta, + c + ); + return BLIS_SUCCESS; + } + + //update the pointer math if matrix B needs to be transposed. + if (bli_obj_has_trans( b )) + { + tb_inc_col = 1; //switch row and column strides + tb_inc_row = ldb; + } + + if ((N <= 3) || ((MR * K) > F_SCRATCH_DIM)) + { + required_packing_A = 0; + } + /* + * The computation loop runs for MRxN columns of C matrix, thus + * accessing the MRxK A matrix data and KxNR B matrix data. + * The computation is organized as inner loops of dimension MRxNR. + */ + // Process MR rows of C matrix at a time. + for (row_idx = 0; (row_idx + (MR - 1)) < M; row_idx += MR) + { + + col_idx_start = 0; + tA_packed = A; + row_idx_packed = row_idx; + lda_packed = lda; + + // This is the part of the pack and compute optimization. + // During the first column iteration, we store the accessed A matrix into + // contiguous static memory. This helps to keep te A matrix in Cache and + // aviods the TLB misses. + if (required_packing_A) + { + col_idx = 0; + + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + tA_packed = A_pack; + +#if 0//def BLIS_ENABLE_PREFETCH + _mm_prefetch((char*)(tC + 0), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 16), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc + 16), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc + 16), _MM_HINT_T0); +#endif + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm11 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + // This loop is processing MR x K + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + _mm256_storeu_ps(tA_packed, ymm3); // the packing of matrix A + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + _mm256_storeu_ps(tA_packed + 8, ymm3); // the packing of matrix A + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_ps(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_ps(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + _mm256_storeu_ps(tA_packed + 16, ymm3); // the packing of matrix A + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_ps(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_ps(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_ps(ymm2, ymm3, ymm14); + + ymm3 = _mm256_loadu_ps(tA + 24); + _mm256_storeu_ps(tA_packed + 24, ymm3); // the packing of matrix A + // ymm7 += ymm0 * ymm3; + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + // ymm11 += ymm1 * ymm3; + ymm11 = _mm256_fmadd_ps(ymm1, ymm3, ymm11); + // ymm15 += ymm2 * ymm3; + ymm15 = _mm256_fmadd_ps(ymm2, ymm3, ymm15); + + tA += lda; + tA_packed += MR; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm11 = _mm256_mul_ps(ymm11, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + ymm15 = _mm256_mul_ps(ymm15, ymm0); + + // multiply C by beta and accumulate col 1. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + _mm256_storeu_ps(tC + 16, ymm6); + _mm256_storeu_ps(tC + 24, ymm7); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm11 = _mm256_fmadd_ps(ymm2, ymm1, ymm11);*/ + _mm256_storeu_ps(tC, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + _mm256_storeu_ps(tC + 24, ymm11); + + // multiply C by beta and accumulate, col 3. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm15 = _mm256_fmadd_ps(ymm2, ymm1, ymm15);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + _mm256_storeu_ps(tC + 24, ymm15); + + // modify the pointer arithematic to use packed A matrix. + col_idx_start = NR; + tA_packed = A_pack; + row_idx_packed = 0; + lda_packed = MR; + } + // Process NR columns of C matrix at a time. + for (col_idx = col_idx_start; (col_idx + (NR - 1)) < N; col_idx += NR) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = tA_packed + row_idx_packed; + +#if 0//def BLIS_ENABLE_PREFETCH + _mm_prefetch((char*)(tC + 0), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 16), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc + 16), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc + 16), _MM_HINT_T0); +#endif + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm11 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + // This loop is processing MR x K + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_ps(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_ps(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_ps(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_ps(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_ps(ymm2, ymm3, ymm14); + + ymm3 = _mm256_loadu_ps(tA + 24); + // ymm7 += ymm0 * ymm3; + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + // ymm11 += ymm1 * ymm3; + ymm11 = _mm256_fmadd_ps(ymm1, ymm3, ymm11); + // ymm15 += ymm2 * ymm3; + ymm15 = _mm256_fmadd_ps(ymm2, ymm3, ymm15); + + tA += lda_packed; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm11 = _mm256_mul_ps(ymm11, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + ymm15 = _mm256_mul_ps(ymm15, ymm0); + + // multiply C by beta and accumulate col 1. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + _mm256_storeu_ps(tC + 16, ymm6); + _mm256_storeu_ps(tC + 24, ymm7); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm11 = _mm256_fmadd_ps(ymm2, ymm1, ymm11);*/ + _mm256_storeu_ps(tC, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + _mm256_storeu_ps(tC + 24, ymm11); + + // multiply C by beta and accumulate, col 3. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm15 = _mm256_fmadd_ps(ymm2, ymm1, ymm15);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + _mm256_storeu_ps(tC + 24, ymm15); + + } + n_remainder = N - col_idx; + + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm11 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm8 = _mm256_fmadd_ps(ymm0, ymm3, ymm8); + ymm12 = _mm256_fmadd_ps(ymm1, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm9 = _mm256_fmadd_ps(ymm0, ymm3, ymm9); + ymm13 = _mm256_fmadd_ps(ymm1, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + ymm14 = _mm256_fmadd_ps(ymm1, ymm3, ymm14); + + ymm3 = _mm256_loadu_ps(tA + 24); + ymm11 = _mm256_fmadd_ps(ymm0, ymm3, ymm11); + ymm15 = _mm256_fmadd_ps(ymm1, ymm3, ymm15); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm11 = _mm256_mul_ps(ymm11, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + ymm15 = _mm256_mul_ps(ymm15, ymm0); + + // multiply C by beta and accumulate, col 1. + /*ymm2 = _mm256_loadu_ps(tC + 0); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm11 = _mm256_fmadd_ps(ymm2, ymm1, ymm11);*/ + _mm256_storeu_ps(tC + 0, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + _mm256_storeu_ps(tC + 24, ymm11); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm15 = _mm256_fmadd_ps(ymm2, ymm1, ymm15);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + _mm256_storeu_ps(tC + 24, ymm15); + + col_idx += 2; + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm12 = _mm256_fmadd_ps(ymm0, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + ymm14 = _mm256_fmadd_ps(ymm0, ymm3, ymm14); + + ymm3 = _mm256_loadu_ps(tA + 24); + ymm15 = _mm256_fmadd_ps(ymm0, ymm3, ymm15); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + ymm15 = _mm256_mul_ps(ymm15, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC + 0); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm15 = _mm256_fmadd_ps(ymm2, ymm1, ymm15);*/ + + _mm256_storeu_ps(tC + 0, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + _mm256_storeu_ps(tC + 24, ymm15); + } + } + + m_remainder = M - row_idx; + + if (m_remainder >= 24) + { + m_remainder -= 24; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_ps(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_ps(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_ps(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_ps(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_ps(ymm2, ymm3, ymm14); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + _mm256_storeu_ps(tC + 16, ymm6); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10);*/ + _mm256_storeu_ps(tC, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm8 = _mm256_fmadd_ps(ymm0, ymm3, ymm8); + ymm12 = _mm256_fmadd_ps(ymm1, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm9 = _mm256_fmadd_ps(ymm0, ymm3, ymm9); + ymm13 = _mm256_fmadd_ps(ymm1, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + ymm14 = _mm256_fmadd_ps(ymm1, ymm3, ymm14); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC + 0); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10);*/ + _mm256_storeu_ps(tC + 0, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + + col_idx += 2; + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm12 = _mm256_fmadd_ps(ymm0, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + ymm14 = _mm256_fmadd_ps(ymm0, ymm3, ymm14); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC + 0); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14);*/ + + _mm256_storeu_ps(tC + 0, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + } + + row_idx += 24; + } + + if (m_remainder >= 16) + { + m_remainder -= 16; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm6 = _mm256_fmadd_ps(ymm1, ymm3, ymm6); + ymm8 = _mm256_fmadd_ps(ymm2, ymm3, ymm8); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(tC, ymm6); + _mm256_storeu_ps(tC + 8, ymm7); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9);*/ + _mm256_storeu_ps(tC, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm6 = _mm256_fmadd_ps(ymm1, ymm3, ymm6); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(tC, ymm6); + _mm256_storeu_ps(tC + 8, ymm7); + + col_idx += 2; + + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + + } + + row_idx += 16; + } + + if (m_remainder >= 8) + { + m_remainder -= 8; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_ps(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_ps(ymm2, ymm3, ymm6); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4);*/ + _mm256_storeu_ps(tC, ymm4); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6);*/ + _mm256_storeu_ps(tC, ymm6); + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_ps(ymm1, ymm3, ymm5); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4);*/ + _mm256_storeu_ps(tC, ymm4); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm5); + + col_idx += 2; + + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + ymm4 = _mm256_mul_ps(ymm4, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4);*/ + _mm256_storeu_ps(tC, ymm4); + + } + + row_idx += 8; + } + // M is not a multiple of 32. + // The handling of edge case where the remainder + // dimension is less than 8. The padding takes place + // to handle this case. + if ((m_remainder) && (lda > 7)) + { + float f_temp[8]; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm5 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(f_temp, ymm7); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9);*/ + _mm256_storeu_ps(f_temp, ymm9); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm5 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + ymm3 = _mm256_loadu_ps(tA); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + + tA += lda; + } + + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(f_temp, ymm7); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm5 = _mm256_setzero_ps(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + ymm3 = _mm256_loadu_ps(tA); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + + tA += lda; + } + + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + // multiply C by beta and accumulate. + ymm5 = _mm256_mul_ps(ymm5, ymm0); + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + m_remainder = 0; + } + + if (m_remainder) + { + float result; + for (; row_idx < M; row_idx += 1) + { + for (col_idx = 0; col_idx < N; col_idx += 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + result = 0; + for (k = 0; k < K; ++k) + { + result += (*tA) * (*tB); + tA += lda; + tB += tb_inc_row; + } + + result *= (*alpha_cast); + (*tC) = /*(*tC) * (*beta_cast) + */result; + } + } + } + + //copy/compute sryk values back to C using SIMD + if ( bli_seq0( *beta_cast ) ) + {//just copy in case of beta = 0 + dim_t _i, _j, k, _l; + if(bli_obj_is_lower(c)) // c is lower + { + //first column + _j = 0; + k = M >> 3; + _i = 0; + for ( _l = 0; _l < k; _l++ ) + { + ymm0 = _mm256_loadu_ps((C + _i*rsc)); + _mm256_storeu_ps((matCbuf + _i*rs_matC), ymm0); + _i += 8; + } + while (_i < M ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + while ( _j < N ) //next column + { + //k = (_j + (8 - (_j & 7))); + _l = _j & 7; + k = (_l != 0) ? (_j + (8 - _l)) : _j; + k = (k <= M) ? k : M; + for ( _i = _j; _i < k; ++_i ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + k = (M - _i) >> 3; + _l = 0; + while ( _l < k ) + { + ymm0 = _mm256_loadu_ps((C + _i*rsc + _j*ldc)); + _mm256_storeu_ps((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + + _i += 8; + _l++; + } + while (_i < M ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + { + k = (_j + 1) >> 3; + _i = 0; + _l = 0; + while ( _l < k ) + { + ymm0 = _mm256_loadu_ps((C + _i*rsc + _j*ldc)); + _mm256_storeu_ps((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + _i += 8; + _l++; + } + while (_i <= _j ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + ++_i; + } + } + } + } + else + {//when beta is non-zero, fmadd and store the results + dim_t _i, _j, k, _l; + ymm1 = _mm256_broadcast_ss(beta_cast); + if(bli_obj_is_lower(c)) //c is lower + { + //first column + _j = 0; + k = M >> 3; + _i = 0; + for ( _l = 0; _l < k; _l++ ) + { + ymm2 = _mm256_loadu_ps((matCbuf + _i*rs_matC)); + ymm0 = _mm256_loadu_ps((C + _i*rsc)); + ymm0 = _mm256_fmadd_ps(ymm2, ymm1, ymm0); + _mm256_storeu_ps((matCbuf + _i*rs_matC), ymm0); + _i += 8; + } + while (_i < M ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + while ( _j < N ) //next column + { + //k = (_j + (8 - (_j & 7))); + _l = _j & 7; + k = (_l != 0) ? (_j + (8 - _l)) : _j; + k = (k <= M) ? k : M; + for ( _i = _j; _i < k; ++_i ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + k = (M - _i) >> 3; + _l = 0; + while ( _l < k ) + { + ymm2 = _mm256_loadu_ps((matCbuf + _i*rs_matC + _j*ldc_matC)); + ymm0 = _mm256_loadu_ps((C + _i*rsc + _j*ldc)); + ymm0 = _mm256_fmadd_ps(ymm2, ymm1, ymm0); + _mm256_storeu_ps((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + + _i += 8; + _l++; + } + while (_i < M ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + { + k = (_j + 1) >> 3; + _i = 0; + _l = 0; + while ( _l < k ) + { + ymm2 = _mm256_loadu_ps((matCbuf + _i*rs_matC + _j*ldc_matC)); + ymm0 = _mm256_loadu_ps((C + _i*rsc + _j*ldc)); + ymm0 = _mm256_fmadd_ps(ymm2, ymm1, ymm0); + _mm256_storeu_ps((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + _i += 8; + _l++; + } + while (_i <= _j ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + ++_i; + } + } + } + } + + return BLIS_SUCCESS; + } + else + return BLIS_NONCONFORMAL_DIMENSIONS; + + +}; + +static err_t bli_dsyrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + int M = bli_obj_length( c ); // number of rows of Matrix C + int N = bli_obj_width( c ); // number of columns of Matrix C + int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + int L = M * N; + + // If alpha is zero, scale by beta and return. + if ((((L) < (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES)) + || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) + { + + int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + int ldc_matC = bli_obj_col_stride( c ); // column stride of matrix C + int ldc = M;//bli_obj_col_stride( c ); // column stride of static buffer for matrix C + int row_idx, col_idx, k; + int rs_matC = bli_obj_row_stride( c ); + int rsc = 1; + double *A = a->buffer; // pointer to elements of Matrix A + double *B = b->buffer; // pointer to elements of Matrix B + double *C = D_C_pack; // pointer to elements of Matrix C + double *matCbuf = c->buffer; + + double *tA = A, *tB = B, *tC = C;//, *tA_pack; + double *tA_packed; // temprorary pointer to hold packed A memory pointer + int row_idx_packed; //packed A memory row index + int lda_packed; //lda of packed A + int col_idx_start; //starting index after A matrix is packed. + dim_t tb_inc_row = 1; // row stride of matrix B + dim_t tb_inc_col = ldb; // column stride of matrix B + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm0, ymm1, ymm2, ymm3; + + int n_remainder; // If the N is non multiple of 3.(N%3) + int m_remainder; // If the M is non multiple of 16.(M%16) + + double *alpha_cast, *beta_cast; // alpha, beta multiples + alpha_cast = (alpha->buffer); + beta_cast = (beta->buffer); + int required_packing_A = 1; + + // when N is equal to 1 call GEMV instead of SYRK + if (N == 1) + { + bli_gemv + ( + alpha, + a, + b, + beta, + c + ); + return BLIS_SUCCESS; + } + + //update the pointer math if matrix B needs to be transposed. + if (bli_obj_has_trans( b )) + { + tb_inc_col = 1; //switch row and column strides + tb_inc_row = ldb; + } + + if ((N <= 3) || ((D_MR * K) > D_SCRATCH_DIM)) + { + required_packing_A = 0; + } + /* + * The computation loop runs for D_MRxN columns of C matrix, thus + * accessing the D_MRxK A matrix data and KxNR B matrix data. + * The computation is organized as inner loops of dimension D_MRxNR. + */ + // Process D_MR rows of C matrix at a time. + for (row_idx = 0; (row_idx + (D_MR - 1)) < M; row_idx += D_MR) + { + + col_idx_start = 0; + tA_packed = A; + row_idx_packed = row_idx; + lda_packed = lda; + + // This is the part of the pack and compute optimization. + // During the first column iteration, we store the accessed A matrix into + // contiguous static memory. This helps to keep te A matrix in Cache and + // aviods the TLB misses. + if (required_packing_A) + { + col_idx = 0; + + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + tA_packed = D_A_pack; + +#if 0//def BLIS_ENABLE_PREFETCH + _mm_prefetch((char*)(tC + 0), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 8), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc + 8), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc + 8), _MM_HINT_T0); +#endif + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + // This loop is processing D_MR x K + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + _mm256_storeu_pd(tA_packed, ymm3); // the packing of matrix A + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + _mm256_storeu_pd(tA_packed + 4, ymm3); // the packing of matrix A + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_pd(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_pd(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + _mm256_storeu_pd(tA_packed + 8, ymm3); // the packing of matrix A + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_pd(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_pd(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_pd(ymm2, ymm3, ymm14); + + ymm3 = _mm256_loadu_pd(tA + 12); + _mm256_storeu_pd(tA_packed + 12, ymm3); // the packing of matrix A + // ymm7 += ymm0 * ymm3; + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + // ymm11 += ymm1 * ymm3; + ymm11 = _mm256_fmadd_pd(ymm1, ymm3, ymm11); + // ymm15 += ymm2 * ymm3; + ymm15 = _mm256_fmadd_pd(ymm2, ymm3, ymm15); + + tA += lda; + tA_packed += D_MR; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm11 = _mm256_mul_pd(ymm11, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + // multiply C by beta and accumulate col 1. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + _mm256_storeu_pd(tC + 8, ymm6); + _mm256_storeu_pd(tC + 12, ymm7); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm11 = _mm256_fmadd_pd(ymm2, ymm1, ymm11);*/ + _mm256_storeu_pd(tC, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + _mm256_storeu_pd(tC + 12, ymm11); + + // multiply C by beta and accumulate, col 3. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm15 = _mm256_fmadd_pd(ymm2, ymm1, ymm15);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + _mm256_storeu_pd(tC + 12, ymm15); + + // modify the pointer arithematic to use packed A matrix. + col_idx_start = NR; + tA_packed = D_A_pack; + row_idx_packed = 0; + lda_packed = D_MR; + } + // Process NR columns of C matrix at a time. + for (col_idx = col_idx_start; (col_idx + (NR - 1)) < N; col_idx += NR) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = tA_packed + row_idx_packed; + +#if 0//def BLIS_ENABLE_PREFETCH + _mm_prefetch((char*)(tC + 0), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 8), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc + 8), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc + 8), _MM_HINT_T0); +#endif + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + // This loop is processing D_MR x K + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_pd(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_pd(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_pd(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_pd(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_pd(ymm2, ymm3, ymm14); + + ymm3 = _mm256_loadu_pd(tA + 12); + // ymm7 += ymm0 * ymm3; + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + // ymm11 += ymm1 * ymm3; + ymm11 = _mm256_fmadd_pd(ymm1, ymm3, ymm11); + // ymm15 += ymm2 * ymm3; + ymm15 = _mm256_fmadd_pd(ymm2, ymm3, ymm15); + + tA += lda_packed; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm11 = _mm256_mul_pd(ymm11, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + // multiply C by beta and accumulate col 1. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + _mm256_storeu_pd(tC + 8, ymm6); + _mm256_storeu_pd(tC + 12, ymm7); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm11 = _mm256_fmadd_pd(ymm2, ymm1, ymm11);*/ + _mm256_storeu_pd(tC, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + _mm256_storeu_pd(tC + 12, ymm11); + + // multiply C by beta and accumulate, col 3. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm15 = _mm256_fmadd_pd(ymm2, ymm1, ymm15);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + _mm256_storeu_pd(tC + 12, ymm15); + + } + n_remainder = N - col_idx; + + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm8 = _mm256_fmadd_pd(ymm0, ymm3, ymm8); + ymm12 = _mm256_fmadd_pd(ymm1, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm9 = _mm256_fmadd_pd(ymm0, ymm3, ymm9); + ymm13 = _mm256_fmadd_pd(ymm1, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + ymm14 = _mm256_fmadd_pd(ymm1, ymm3, ymm14); + + ymm3 = _mm256_loadu_pd(tA + 12); + ymm11 = _mm256_fmadd_pd(ymm0, ymm3, ymm11); + ymm15 = _mm256_fmadd_pd(ymm1, ymm3, ymm15); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm11 = _mm256_mul_pd(ymm11, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + // multiply C by beta and accumulate, col 1. + /*ymm2 = _mm256_loadu_pd(tC + 0); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm11 = _mm256_fmadd_pd(ymm2, ymm1, ymm11);*/ + _mm256_storeu_pd(tC + 0, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + _mm256_storeu_pd(tC + 12, ymm11); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm15 = _mm256_fmadd_pd(ymm2, ymm1, ymm15);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + _mm256_storeu_pd(tC + 12, ymm15); + + col_idx += 2; + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm12 = _mm256_fmadd_pd(ymm0, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + ymm14 = _mm256_fmadd_pd(ymm0, ymm3, ymm14); + + ymm3 = _mm256_loadu_pd(tA + 12); + ymm15 = _mm256_fmadd_pd(ymm0, ymm3, ymm15); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC + 0); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm15 = _mm256_fmadd_pd(ymm2, ymm1, ymm15);*/ + + _mm256_storeu_pd(tC + 0, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + _mm256_storeu_pd(tC + 12, ymm15); + } + } + + m_remainder = M - row_idx; + + if (m_remainder >= 12) + { + m_remainder -= 12; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_pd(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_pd(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_pd(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_pd(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_pd(ymm2, ymm3, ymm14); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + _mm256_storeu_pd(tC + 8, ymm6); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10);*/ + _mm256_storeu_pd(tC, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm8 = _mm256_fmadd_pd(ymm0, ymm3, ymm8); + ymm12 = _mm256_fmadd_pd(ymm1, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm9 = _mm256_fmadd_pd(ymm0, ymm3, ymm9); + ymm13 = _mm256_fmadd_pd(ymm1, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + ymm14 = _mm256_fmadd_pd(ymm1, ymm3, ymm14); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC + 0); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10);*/ + _mm256_storeu_pd(tC + 0, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + + col_idx += 2; + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm12 = _mm256_fmadd_pd(ymm0, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + ymm14 = _mm256_fmadd_pd(ymm0, ymm3, ymm14); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC + 0); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14);*/ + + _mm256_storeu_pd(tC + 0, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + } + + row_idx += 12; + } + + if (m_remainder >= 8) + { + m_remainder -= 8; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm6 = _mm256_fmadd_pd(ymm1, ymm3, ymm6); + ymm8 = _mm256_fmadd_pd(ymm2, ymm3, ymm8); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(tC, ymm6); + _mm256_storeu_pd(tC + 4, ymm7); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9);*/ + _mm256_storeu_pd(tC, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm6 = _mm256_fmadd_pd(ymm1, ymm3, ymm6); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(tC, ymm6); + _mm256_storeu_pd(tC + 4, ymm7); + + col_idx += 2; + + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + + } + + row_idx += 8; + } + + if (m_remainder >= 4) + { + m_remainder -= 4; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_pd(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_pd(ymm2, ymm3, ymm6); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4);*/ + _mm256_storeu_pd(tC, ymm4); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6);*/ + _mm256_storeu_pd(tC, ymm6); + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_pd(ymm1, ymm3, ymm5); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4);*/ + _mm256_storeu_pd(tC, ymm4); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm5); + + col_idx += 2; + + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + ymm4 = _mm256_mul_pd(ymm4, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4);*/ + _mm256_storeu_pd(tC, ymm4); + + } + + row_idx += 4; + } + // M is not a multiple of 32. + // The handling of edge case where the remainder + // dimension is less than 8. The padding takes place + // to handle this case. + if ((m_remainder) && (lda > 3)) + { + double f_temp[8]; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm5 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(f_temp, ymm7); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9);*/ + _mm256_storeu_pd(f_temp, ymm9); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm5 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + ymm3 = _mm256_loadu_pd(tA); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + + tA += lda; + } + + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(f_temp, ymm7); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm5 = _mm256_setzero_pd(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + ymm3 = _mm256_loadu_pd(tA); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + + tA += lda; + } + + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + // multiply C by beta and accumulate. + ymm5 = _mm256_mul_pd(ymm5, ymm0); + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + m_remainder = 0; + } + + if (m_remainder) + { + double result; + for (; row_idx < M; row_idx += 1) + { + for (col_idx = 0; col_idx < N; col_idx += 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + result = 0; + for (k = 0; k < K; ++k) + { + result += (*tA) * (*tB); + tA += lda; + tB += tb_inc_row; + } + + result *= (*alpha_cast); + (*tC) = /*(*tC) * (*beta_cast) + */result; + } + } + } + + //copy/compute sryk values back to C using SIMD + if ( bli_seq0( *beta_cast ) ) + {//just copy for beta = 0 + dim_t _i, _j, k, _l; + if(bli_obj_is_lower(c)) //c is lower + { + //first column + _j = 0; + k = M >> 2; + _i = 0; + for ( _l = 0; _l < k; _l++ ) + { + ymm0 = _mm256_loadu_pd((C + _i*rsc)); + _mm256_storeu_pd((matCbuf + _i*rs_matC), ymm0); + _i += 4; + } + while (_i < M ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + while ( _j < N ) //next column + { + //k = (_j + (4 - (_j & 3))); + _l = _j & 3; + k = (_l != 0) ? (_j + (4 - _l)) : _j; + k = (k <= M) ? k : M; + for ( _i = _j; _i < k; ++_i ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + k = (M - _i) >> 2; + _l = 0; + while ( _l < k ) + { + ymm0 = _mm256_loadu_pd((C + _i*rsc + _j*ldc)); + _mm256_storeu_pd((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + + _i += 4; + _l++; + } + while (_i < M ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + { + k = (_j + 1) >> 2; + _i = 0; + _l = 0; + while ( _l < k ) + { + ymm0 = _mm256_loadu_pd((C + _i*rsc + _j*ldc)); + _mm256_storeu_pd((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + _i += 4; + _l++; + } + while (_i <= _j ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + ++_i; + } + } + } + } + else + {//when beta is non-zero, fmadd and store the results + dim_t _i, _j, k, _l; + ymm1 = _mm256_broadcast_sd(beta_cast); + if(bli_obj_is_lower(c)) //c is lower + { + //first column + _j = 0; + k = M >> 2; + _i = 0; + for ( _l = 0; _l < k; _l++ ) + { + ymm2 = _mm256_loadu_pd((matCbuf + _i*rs_matC)); + ymm0 = _mm256_loadu_pd((C + _i*rsc)); + ymm0 = _mm256_fmadd_pd(ymm2, ymm1, ymm0); + _mm256_storeu_pd((matCbuf + _i*rs_matC), ymm0); + _i += 4; + } + while (_i < M ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + while ( _j < N ) //next column + { + //k = (_j + (4 - (_j & 3))); + _l = _j & 3; + k = (_l != 0) ? (_j + (4 - _l)) : _j; + k = (k <= M) ? k : M; + for ( _i = _j; _i < k; ++_i ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + k = (M - _i) >> 2; + _l = 0; + while ( _l < k ) + { + ymm2 = _mm256_loadu_pd((matCbuf + _i*rs_matC + _j*ldc_matC)); + ymm0 = _mm256_loadu_pd((C + _i*rsc + _j*ldc)); + ymm0 = _mm256_fmadd_pd(ymm2, ymm1, ymm0); + _mm256_storeu_pd((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + + _i += 4; + _l++; + } + while (_i < M ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + { + k = (_j + 1) >> 2; + _i = 0; + _l = 0; + while ( _l < k ) + { + ymm2 = _mm256_loadu_pd((matCbuf + _i*rs_matC + _j*ldc_matC)); + ymm0 = _mm256_loadu_pd((C + _i*rsc + _j*ldc)); + ymm0 = _mm256_fmadd_pd(ymm2, ymm1, ymm0); + _mm256_storeu_pd((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + _i += 4; + _l++; + } + while (_i <= _j ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + ++_i; + } + } + } + } + + return BLIS_SUCCESS; + } + else + return BLIS_NONCONFORMAL_DIMENSIONS; + + +}; + +static err_t bli_ssyrk_small_atbn + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int M = bli_obj_length(c); // number of rows of Matrix C + int N = bli_obj_width(c); // number of columns of Matrix C + int K = bli_obj_length(b); // number of rows of Matrix B + int lda = bli_obj_col_stride(a); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + int ldb = bli_obj_col_stride(b); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + int ldc_matC = bli_obj_col_stride( c ); // column stride of matrix C + int ldc = M;//bli_obj_col_stride( c ); // column stride of static buffer for matrix C + int row_idx = 0, col_idx = 0, k; + int rs_matC = bli_obj_row_stride( c ); + int rsc = 1; + float *A = a->buffer; // pointer to matrix A elements, stored in row major format + float *B = b->buffer; // pointer to matrix B elements, stored in column major format + float *C = C_pack; // pointer to matrix C elements, stored in column major format + float *matCbuf = c->buffer; + + float *tA = A, *tB = B, *tC = C; + + __m256 ymm4, ymm5, ymm6, ymm7; + __m256 ymm8, ymm9, ymm10, ymm11; + __m256 ymm12, ymm13, ymm14, ymm15; + __m256 ymm0, ymm1, ymm2, ymm3; + + float result, scratch[8]; + float *alpha_cast, *beta_cast; // alpha, beta multiples + alpha_cast = (alpha->buffer); + beta_cast = (beta->buffer); + + // The non-copy version of the A^T SYRK gives better performance for the small M cases. + // The threshold is controlled by BLIS_ATBN_M_THRES + if (M <= BLIS_ATBN_M_THRES) + { + for (col_idx = 0; (col_idx + (NR - 1)) < N; col_idx += NR) + { + for (row_idx = 0; (row_idx + (AT_MR - 1)) < M; row_idx += AT_MR) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm11 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + //The inner loop computes the 4x3 values of the matrix. + //The computation pattern is: + // ymm4 ymm5 ymm6 + // ymm7 ymm8 ymm9 + // ymm10 ymm11 ymm12 + // ymm13 ymm14 ymm15 + + //The Dot operation is performed in the inner loop, 8 float elements fit + //in the YMM register hence loop count incremented by 8 + for (k = 0; (k + 7) < K; k += 8) + { + ymm0 = _mm256_loadu_ps(tB + 0); + ymm1 = _mm256_loadu_ps(tB + ldb); + ymm2 = _mm256_loadu_ps(tB + 2 * ldb); + + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_ps(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_ps(ymm2, ymm3, ymm6); + + ymm3 = _mm256_loadu_ps(tA + lda); + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + ymm3 = _mm256_loadu_ps(tA + 2 * lda); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + ymm11 = _mm256_fmadd_ps(ymm1, ymm3, ymm11); + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 3 * lda); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + ymm14 = _mm256_fmadd_ps(ymm1, ymm3, ymm14); + ymm15 = _mm256_fmadd_ps(ymm2, ymm3, ymm15); + + tA += 8; + tB += 8; + + } + + // if K is not a multiple of 8, padding is done before load using temproary array. + if (k < K) + { + int iter; + float data_feeder[8] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_ps(data_feeder); + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter + ldb]; + ymm1 = _mm256_loadu_ps(data_feeder); + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter + 2 * ldb]; + ymm2 = _mm256_loadu_ps(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_ps(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_ps(ymm2, ymm3, ymm6); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[2 * lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + ymm11 = _mm256_fmadd_ps(ymm1, ymm3, ymm11); + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[3 * lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + ymm14 = _mm256_fmadd_ps(ymm1, ymm3, ymm14); + ymm15 = _mm256_fmadd_ps(ymm2, ymm3, ymm15); + + } + + //horizontal addition and storage of the data. + //Results for 4x3 blocks of C is stored here + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + _mm256_storeu_ps(scratch, ymm4); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm7 = _mm256_hadd_ps(ymm7, ymm7); + ymm7 = _mm256_hadd_ps(ymm7, ymm7); + _mm256_storeu_ps(scratch, ymm7); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm10 = _mm256_hadd_ps(ymm10, ymm10); + ymm10 = _mm256_hadd_ps(ymm10, ymm10); + _mm256_storeu_ps(scratch, ymm10); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm13 = _mm256_hadd_ps(ymm13, ymm13); + ymm13 = _mm256_hadd_ps(ymm13, ymm13); + _mm256_storeu_ps(scratch, ymm13); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + tC += ldc; + ymm5 = _mm256_hadd_ps(ymm5, ymm5); + ymm5 = _mm256_hadd_ps(ymm5, ymm5); + _mm256_storeu_ps(scratch, ymm5); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm8 = _mm256_hadd_ps(ymm8, ymm8); + ymm8 = _mm256_hadd_ps(ymm8, ymm8); + _mm256_storeu_ps(scratch, ymm8); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm11 = _mm256_hadd_ps(ymm11, ymm11); + ymm11 = _mm256_hadd_ps(ymm11, ymm11); + _mm256_storeu_ps(scratch, ymm11); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm14 = _mm256_hadd_ps(ymm14, ymm14); + ymm14 = _mm256_hadd_ps(ymm14, ymm14); + _mm256_storeu_ps(scratch, ymm14); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + tC += ldc; + ymm6 = _mm256_hadd_ps(ymm6, ymm6); + ymm6 = _mm256_hadd_ps(ymm6, ymm6); + _mm256_storeu_ps(scratch, ymm6); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm9 = _mm256_hadd_ps(ymm9, ymm9); + ymm9 = _mm256_hadd_ps(ymm9, ymm9); + _mm256_storeu_ps(scratch, ymm9); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm12 = _mm256_hadd_ps(ymm12, ymm12); + ymm12 = _mm256_hadd_ps(ymm12, ymm12); + _mm256_storeu_ps(scratch, ymm12); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm15 = _mm256_hadd_ps(ymm15, ymm15); + ymm15 = _mm256_hadd_ps(ymm15, ymm15); + _mm256_storeu_ps(scratch, ymm15); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + } + } + + int processed_col = col_idx; + int processed_row = row_idx; + + //The edge case handling where N is not a multiple of 3 + if (processed_col < N) + { + for (col_idx = processed_col; col_idx < N; col_idx += 1) + { + for (row_idx = 0; (row_idx + (AT_MR - 1)) < M; row_idx += AT_MR) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + + //The inner loop computes the 4x1 values of the matrix. + //The computation pattern is: + // ymm4 + // ymm7 + // ymm10 + // ymm13 + + for (k = 0; (k + 7) < K; k += 8) + { + ymm0 = _mm256_loadu_ps(tB + 0); + + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + ymm3 = _mm256_loadu_ps(tA + lda); + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + + ymm3 = _mm256_loadu_ps(tA + 2 * lda); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + + ymm3 = _mm256_loadu_ps(tA + 3 * lda); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + + tA += 8; + tB += 8; + } + + // if K is not a multiple of 8, padding is done before load using temproary array. + if (k < K) + { + int iter; + float data_feeder[8] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_ps(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[2 * lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[3 * lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + + } + + //horizontal addition and storage of the data. + //Results for 4x1 blocks of C is stored here + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + _mm256_storeu_ps(scratch, ymm4); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm7 = _mm256_hadd_ps(ymm7, ymm7); + ymm7 = _mm256_hadd_ps(ymm7, ymm7); + _mm256_storeu_ps(scratch, ymm7); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm10 = _mm256_hadd_ps(ymm10, ymm10); + ymm10 = _mm256_hadd_ps(ymm10, ymm10); + _mm256_storeu_ps(scratch, ymm10); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm13 = _mm256_hadd_ps(ymm13, ymm13); + ymm13 = _mm256_hadd_ps(ymm13, ymm13); + _mm256_storeu_ps(scratch, ymm13); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + } + } + processed_row = row_idx; + } + + //The edge case handling where M is not a multiple of 4 + if (processed_row < M) + { + for (row_idx = processed_row; row_idx < M; row_idx += 1) + { + for (col_idx = 0; col_idx < N; col_idx += 1) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + + for (k = 0; (k + 7) < K; k += 8) + { + ymm0 = _mm256_loadu_ps(tB + 0); + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + tA += 8; + tB += 8; + } + + // if K is not a multiple of 8, padding is done before load using temproary array. + if (k < K) + { + int iter; + float data_feeder[8] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_ps(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + } + + //horizontal addition and storage of the data. + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + _mm256_storeu_ps(scratch, ymm4); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + } + } + } + + //copy/compute sryk values back to C + if ( bli_seq0( *beta_cast ) ) //when beta is 0, just copy result to C + { + dim_t _i, _j; + if(bli_obj_is_lower(c)) //c is lower + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i <= 0 ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i >= 0 ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + } + else //when beta is non-zero, multiply and store result to C + { + dim_t _i, _j; + if(bli_obj_is_lower(c)) //c is lower + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i <= 0 ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i >= 0 ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + } + + return BLIS_SUCCESS; + } + else + return BLIS_NONCONFORMAL_DIMENSIONS; +} + +static err_t bli_dsyrk_small_atbn + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int M = bli_obj_length( c ); // number of rows of Matrix C + int N = bli_obj_width( c ); // number of columns of Matrix C + int K = bli_obj_length( b ); // number of rows of Matrix B + int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + int ldc_matC = bli_obj_col_stride( c ); // column stride of matrix C + int ldc = M;//bli_obj_col_stride( c ); // column stride of static buffer for matrix C + int row_idx = 0, col_idx = 0, k; + int rs_matC = bli_obj_row_stride( c ); + int rsc = 1; + double *A = a->buffer; // pointer to matrix A elements, stored in row major format + double *B = b->buffer; // pointer to matrix B elements, stored in column major format + double *C = D_C_pack; // pointer to matrix C elements, stored in column major format + double *matCbuf = c->buffer; + + double *tA = A, *tB = B, *tC = C; + + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm0, ymm1, ymm2, ymm3; + + double result, scratch[8]; + double *alpha_cast, *beta_cast; // alpha, beta multiples + alpha_cast = (alpha->buffer); + beta_cast = (beta->buffer); + + // The non-copy version of the A^T SYRK gives better performance for the small M cases. + // The threshold is controlled by BLIS_ATBN_M_THRES + if (M <= BLIS_ATBN_M_THRES) + { + for (col_idx = 0; (col_idx + (NR - 1)) < N; col_idx += NR) + { + for (row_idx = 0; (row_idx + (AT_MR - 1)) < M; row_idx += AT_MR) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + //The inner loop computes the 4x3 values of the matrix. + //The computation pattern is: + // ymm4 ymm5 ymm6 + // ymm7 ymm8 ymm9 + // ymm10 ymm11 ymm12 + // ymm13 ymm14 ymm15 + + //The Dot operation is performed in the inner loop, 4 double elements fit + //in the YMM register hence loop count incremented by 4 + for (k = 0; (k + 3) < K; k += 4) + { + ymm0 = _mm256_loadu_pd(tB + 0); + ymm1 = _mm256_loadu_pd(tB + ldb); + ymm2 = _mm256_loadu_pd(tB + 2 * ldb); + + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_pd(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_pd(ymm2, ymm3, ymm6); + + ymm3 = _mm256_loadu_pd(tA + lda); + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + ymm3 = _mm256_loadu_pd(tA + 2 * lda); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + ymm11 = _mm256_fmadd_pd(ymm1, ymm3, ymm11); + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 3 * lda); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + ymm14 = _mm256_fmadd_pd(ymm1, ymm3, ymm14); + ymm15 = _mm256_fmadd_pd(ymm2, ymm3, ymm15); + + tA += 4; + tB += 4; + + } + + // if K is not a multiple of 4, padding is done before load using temproary array. + if (k < K) + { + int iter; + double data_feeder[4] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_pd(data_feeder); + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter + ldb]; + ymm1 = _mm256_loadu_pd(data_feeder); + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter + 2 * ldb]; + ymm2 = _mm256_loadu_pd(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_pd(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_pd(ymm2, ymm3, ymm6); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[2 * lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + ymm11 = _mm256_fmadd_pd(ymm1, ymm3, ymm11); + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[3 * lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + ymm14 = _mm256_fmadd_pd(ymm1, ymm3, ymm14); + ymm15 = _mm256_fmadd_pd(ymm2, ymm3, ymm15); + + } + + //horizontal addition and storage of the data. + //Results for 4x3 blocks of C is stored here + ymm4 = _mm256_hadd_pd(ymm4, ymm4); + _mm256_storeu_pd(scratch, ymm4); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm7 = _mm256_hadd_pd(ymm7, ymm7); + _mm256_storeu_pd(scratch, ymm7); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm10 = _mm256_hadd_pd(ymm10, ymm10); + _mm256_storeu_pd(scratch, ymm10); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm13 = _mm256_hadd_pd(ymm13, ymm13); + _mm256_storeu_pd(scratch, ymm13); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + + tC += ldc; + ymm5 = _mm256_hadd_pd(ymm5, ymm5); + _mm256_storeu_pd(scratch, ymm5); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm8 = _mm256_hadd_pd(ymm8, ymm8); + _mm256_storeu_pd(scratch, ymm8); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm11 = _mm256_hadd_pd(ymm11, ymm11); + _mm256_storeu_pd(scratch, ymm11); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm14 = _mm256_hadd_pd(ymm14, ymm14); + _mm256_storeu_pd(scratch, ymm14); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + + tC += ldc; + ymm6 = _mm256_hadd_pd(ymm6, ymm6); + _mm256_storeu_pd(scratch, ymm6); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm9 = _mm256_hadd_pd(ymm9, ymm9); + _mm256_storeu_pd(scratch, ymm9); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm12 = _mm256_hadd_pd(ymm12, ymm12); + _mm256_storeu_pd(scratch, ymm12); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm15 = _mm256_hadd_pd(ymm15, ymm15); + _mm256_storeu_pd(scratch, ymm15); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + } + } + + int processed_col = col_idx; + int processed_row = row_idx; + + //The edge case handling where N is not a multiple of 3 + if (processed_col < N) + { + for (col_idx = processed_col; col_idx < N; col_idx += 1) + { + for (row_idx = 0; (row_idx + (AT_MR - 1)) < M; row_idx += AT_MR) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + + //The inner loop computes the 4x1 values of the matrix. + //The computation pattern is: + // ymm4 + // ymm7 + // ymm10 + // ymm13 + + for (k = 0; (k + 3) < K; k += 4) + { + ymm0 = _mm256_loadu_pd(tB + 0); + + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + ymm3 = _mm256_loadu_pd(tA + lda); + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + + ymm3 = _mm256_loadu_pd(tA + 2 * lda); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + + ymm3 = _mm256_loadu_pd(tA + 3 * lda); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + + tA += 4; + tB += 4; + } + // if K is not a multiple of 4, padding is done before load using temproary array. + if (k < K) + { + int iter; + double data_feeder[4] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_pd(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[2 * lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[3 * lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + + } + + //horizontal addition and storage of the data. + //Results for 4x1 blocks of C is stored here + ymm4 = _mm256_hadd_pd(ymm4, ymm4); + _mm256_storeu_pd(scratch, ymm4); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm7 = _mm256_hadd_pd(ymm7, ymm7); + _mm256_storeu_pd(scratch, ymm7); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm10 = _mm256_hadd_pd(ymm10, ymm10); + _mm256_storeu_pd(scratch, ymm10); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm13 = _mm256_hadd_pd(ymm13, ymm13); + _mm256_storeu_pd(scratch, ymm13); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + } + } + processed_row = row_idx; + } + + // The edge case handling where M is not a multiple of 4 + if (processed_row < M) + { + for (row_idx = processed_row; row_idx < M; row_idx += 1) + { + for (col_idx = 0; col_idx < N; col_idx += 1) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + + for (k = 0; (k + 3) < K; k += 4) + { + ymm0 = _mm256_loadu_pd(tB + 0); + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + tA += 4; + tB += 4; + } + + // if K is not a multiple of 4, padding is done before load using temproary array. + if (k < K) + { + int iter; + double data_feeder[4] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_pd(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + } + + //horizontal addition and storage of the data. + ymm4 = _mm256_hadd_pd(ymm4, ymm4); + _mm256_storeu_pd(scratch, ymm4); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + } + } + } + + //copy/compute sryk values back to C + if ( bli_seq0( *beta_cast ) ) //when beta is 0, just copy result to C + { + dim_t _i, _j; + if(bli_obj_is_lower(c)) //c is lower + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i <= 0 ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i >= 0 ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + } + else //when beta is non-zero, multiply and store result to C + { + dim_t _i, _j; + if(bli_obj_is_lower(c)) //c is lower + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i <= 0 ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i >= 0 ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + } + + return BLIS_SUCCESS; + } + else + return BLIS_NONCONFORMAL_DIMENSIONS; +} + +#endif + From cf9c1150515b8e9cc4f12e0d4787b3471b12ba4a Mon Sep 17 00:00:00 2001 From: kdevraje Date: Thu, 3 Jan 2019 09:51:46 +0530 Subject: [PATCH 19/53] This commit adds a macro, which is to be enabled when BLIS is working on single instance mode Change-Id: I7f3fd654b78e64c4e6e24e9f0e245b1a30c492b0 --- config/zen/bli_cntx_init_zen.c | 56 +++++++++++----------------------- config/zen/bli_family_zen.h | 3 ++ 2 files changed, 20 insertions(+), 39 deletions(-) diff --git a/config/zen/bli_cntx_init_zen.c b/config/zen/bli_cntx_init_zen.c index be0eae894..84f2eb279 100644 --- a/config/zen/bli_cntx_init_zen.c +++ b/config/zen/bli_cntx_init_zen.c @@ -117,56 +117,34 @@ void bli_cntx_init_zen( cntx_t* cntx ) Multi Instance performance improvement of DGEMM when binded to a CCX In Multi instance each thread runs a sequential DGEMM. - a) CPU freq 2.6 Ghz + a) If BLIS is run in a multi instance mode with + CPU freq 2.6/2.2 Ghz DDR4 clock frequency 2400Mhz - Multi instance mode mc = 240, kc = 512, and nc = 2040 - - b) CPU freq 2.2Ghz - DDR4 clock frequency 2400Mhz - Multi Instance mode - either - mc = 240, kc = 512 and nc = 2040 - (or) - mc = 390, kc = 512 and nc = 4080 + has better performance on EPYC server, over the default block sizes. - c) If BLIS is run in Single Instance mode choose + b) If BLIS is run in Single Instance mode mc = 510, kc = 1024 and nc = 4080 */ #ifdef BLIS_ENABLE_ZEN_BLOCK_SIZES - // Zen optmized level 3 cache block sizes - /************************************************************************ - Below block sizes of DGEMM, works better in a multi instance mode, - for clock frequency of 2.2GHz and DDR4 clock frequency of 2400MHz - ************************************************************************/ - bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 240, 144, 72 ); - bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); - bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); - - /*********************************************************************************** - Below block sizes of DGEMM, gives better performance in a multi instance mode, - for clock frequency of 2.6GHz and DDR4 clock frequency of 2400MHz - **************************************************************************************/ - //bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 390, 144, 72 ); - //bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); - //bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); - - /****************************************************************************** - BLIS on single instance mode, gives better perfomance with - below mentioned default block size values - ********************************************************************************/ - //bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 510, 144, 72 ); - //bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 1024, 256, 256 ); - //bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); + #if BLIS_ENABLE_SINGLE_INSTANCE_BLOCK_SIZES + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 510, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 1024, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); + #else + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 240, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); + #endif #else - - bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); - bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); - bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); #endif + //bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); bli_blksz_init_easy( &blkszs[ BLIS_AF ], 8, 8, -1, -1 ); bli_blksz_init_easy( &blkszs[ BLIS_DF ], 8, 8, -1, -1 ); diff --git a/config/zen/bli_family_zen.h b/config/zen/bli_family_zen.h index 743c38f32..281bebf50 100644 --- a/config/zen/bli_family_zen.h +++ b/config/zen/bli_family_zen.h @@ -46,6 +46,7 @@ #define BLIS_ENABLE_SMALL_MATRIX #define BLIS_ENABLE_SMALL_MATRIX_TRSM + // This will select the threshold below which small matrix code will be called. #define BLIS_SMALL_MATRIX_THRES 700 #define BLIS_SMALL_M_RECT_MATRIX_THRES 160 @@ -56,5 +57,7 @@ #define BLIS_SMALL_MATRIX_A_THRES_M_SYRK 96 #define BLIS_SMALL_MATRIX_A_THRES_N_SYRK 128 +//This macro will enable BLIS DGEMM to choose block sizes for a single instance mode +#define BLIS_ENABLE_SINGLE_INSTANCE_BLOCK_SIZES 0 //#endif From cdbf16aa93234e0d6a80f0d0e385ec81e7b75465 Mon Sep 17 00:00:00 2001 From: prangana Date: Fri, 4 Jan 2019 15:59:21 +0530 Subject: [PATCH 20/53] Update version 1.3 Change-Id: I32a7d24af860e87a60396614075236afb65a28a9 --- so_version | 2 +- version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/so_version b/so_version index 15d6c7665..bcd01fc72 100644 --- a/so_version +++ b/so_version @@ -1,2 +1,2 @@ 1 -2.0 +3.0 diff --git a/version b/version index 5625e59da..7e32cd569 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.2 +1.3 From 95e070581c54ed2edc211874faec56055ea298c8 Mon Sep 17 00:00:00 2001 From: mkv Date: Thu, 21 Feb 2019 01:04:16 -0500 Subject: [PATCH 21/53] added test file for initial commit --- test.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 test.txt diff --git a/test.txt b/test.txt new file mode 100644 index 000000000..9daeafb98 --- /dev/null +++ b/test.txt @@ -0,0 +1 @@ +test From 4c7e6680832b497468cf50c2399e3ac4de0e3450 Mon Sep 17 00:00:00 2001 From: praveeng Date: Thu, 21 Feb 2019 12:44:38 +0530 Subject: [PATCH 22/53] deleted test.txt Change-Id: I3871f5fe76e548bc29ec2733745b29964e829dd3 --- test.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 test.txt diff --git a/test.txt b/test.txt deleted file mode 100644 index 9daeafb98..000000000 --- a/test.txt +++ /dev/null @@ -1 +0,0 @@ -test From ed13ad465dcba350ad3d5e16c9cc7542e33f3760 Mon Sep 17 00:00:00 2001 From: mkv Date: Thu, 21 Feb 2019 01:04:16 -0500 Subject: [PATCH 23/53] added test file for initial commit --- test.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 test.txt diff --git a/test.txt b/test.txt new file mode 100644 index 000000000..9daeafb98 --- /dev/null +++ b/test.txt @@ -0,0 +1 @@ +test From e938ff08cea3d108c84524eb129d9e89d701ea90 Mon Sep 17 00:00:00 2001 From: praveeng Date: Thu, 21 Feb 2019 12:44:38 +0530 Subject: [PATCH 24/53] deleted test.txt Change-Id: I3871f5fe76e548bc29ec2733745b29964e829dd3 --- test.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 test.txt diff --git a/test.txt b/test.txt deleted file mode 100644 index 9daeafb98..000000000 --- a/test.txt +++ /dev/null @@ -1 +0,0 @@ -test From f5ed95ecd7d5eb4a63e1333ad5cc6765fc8df9fe Mon Sep 17 00:00:00 2001 From: Kiran Varaganti Date: Tue, 5 Mar 2019 15:01:57 +0530 Subject: [PATCH 25/53] Merged BLIS Release 1.3 Modified config/zen/make_defs.mk, now CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 Change-Id: Ia0942d285a21447cd0c470de1bc021fe63e80d81 --- config/zen/bli_cntx_init_zen.c | 38 +- config/zen/bli_family_zen.h | 13 +- config/zen/make_defs.mk | 6 +- frame/3/syrk/bli_syrk_front.c | 28 +- frame/3/syrk/bli_syrk_front.h | 11 + frame/3/trsm/bli_trsm_front.c | 44 + frame/3/trsm/bli_trsm_front.h | 11 + kernels/zen/1/bli_amaxv_zen_int.c | 14 +- kernels/zen/1/bli_axpyv_zen_int.c | 16 +- kernels/zen/1/bli_axpyv_zen_int10.c | 16 +- kernels/zen/1/bli_dotv_zen_int.c | 16 +- kernels/zen/1/bli_dotv_zen_int10.c | 15 +- kernels/zen/1/bli_dotxv_zen_int.c | 16 +- kernels/zen/3/bli_syrk_small.c | 4186 +++++++ kernels/zen/3/bli_trsm_small.c | 15021 ++++++++++++++++++++++++++ test/Makefile | 2 +- 16 files changed, 19434 insertions(+), 19 deletions(-) create mode 100644 kernels/zen/3/bli_syrk_small.c create mode 100644 kernels/zen/3/bli_trsm_small.c diff --git a/config/zen/bli_cntx_init_zen.c b/config/zen/bli_cntx_init_zen.c index 41de6fd4e..84757ecae 100644 --- a/config/zen/bli_cntx_init_zen.c +++ b/config/zen/bli_cntx_init_zen.c @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -111,15 +112,42 @@ void bli_cntx_init_zen( cntx_t* cntx ) // s d c z bli_blksz_init_easy( &blkszs[ BLIS_MR ], 6, 6, 3, 3 ); bli_blksz_init_easy( &blkszs[ BLIS_NR ], 16, 8, 8, 4 ); + +/* + Multi Instance performance improvement of DGEMM when binded to a CCX + In Multi instance each thread runs a sequential DGEMM. + + a) If BLIS is run in a multi instance mode with + CPU freq 2.6/2.2 Ghz + DDR4 clock frequency 2400Mhz + mc = 240, kc = 512, and nc = 2040 + has better performance on EPYC server, over the default block sizes. + + b) If BLIS is run in Single Instance mode + mc = 510, kc = 1024 and nc = 4080 + +*/ + #ifdef BLIS_ENABLE_ZEN_BLOCK_SIZES // Zen optmized level 3 cache block sizes - bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 510, 144, 72 ); - bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 1024, 256, 256 ); + #if BLIS_ENABLE_SINGLE_INSTANCE_BLOCK_SIZES + + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 510, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 1024, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); + + #else + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 240, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); + #endif #else - bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); - bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); #endif - bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); + + bli_blksz_init_easy( &blkszs[ BLIS_AF ], 8, 8, -1, -1 ); bli_blksz_init_easy( &blkszs[ BLIS_DF ], 8, 8, -1, -1 ); diff --git a/config/zen/bli_family_zen.h b/config/zen/bli_family_zen.h index c872a21eb..d69f5270a 100644 --- a/config/zen/bli_family_zen.h +++ b/config/zen/bli_family_zen.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2016, Advanced Micro Devices, Inc. + Copyright (C) 2018, Advanced Micro Devices, Inc Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -43,14 +43,21 @@ #define BLIS_THREAD_MAX_JR 1 #define BLIS_ENABLE_ZEN_BLOCK_SIZES -//#define BLIS_ENABLE_SMALL_MATRIX +#define BLIS_ENABLE_SMALL_MATRIX +#define BLIS_ENABLE_SMALL_MATRIX_TRSM + // This will select the threshold below which small matrix code will be called. #define BLIS_SMALL_MATRIX_THRES 700 #define BLIS_SMALL_M_RECT_MATRIX_THRES 160 #define BLIS_SMALL_K_RECT_MATRIX_THRES 128 +#define BLIS_SMALL_MATRIX_THRES_TRSM 32768 //128(128+128) => m*(m+n) +#define BLIS_SMALL_MATRIX_A_THRES_TRSM 128 +#define BLIS_SMALL_MATRIX_A_THRES_M_SYRK 96 +#define BLIS_SMALL_MATRIX_A_THRES_N_SYRK 128 - +//This macro will enable BLIS DGEMM to choose block sizes for a single instance mode +#define BLIS_ENABLE_SINGLE_INSTANCE_BLOCK_SIZES 0 //#endif diff --git a/config/zen/make_defs.mk b/config/zen/make_defs.mk index 40b07661b..a485af27b 100644 --- a/config/zen/make_defs.mk +++ b/config/zen/make_defs.mk @@ -64,13 +64,13 @@ endif CKOPTFLAGS := $(COPTFLAGS) ifeq ($(CC_VENDOR),gcc) # gcc 6.0 (clang 4.0) or later: -#CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 +CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 # gcc 4.9 (clang 3.5) or later: # possibly add zen-specific instructions: -mclzero -madx -mrdseed -mmwaitx -msha -mxsavec -mxsaves -mclflushopt -mpopcnt -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +#CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp else ifeq ($(CC_VENDOR),clang) -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-fma4 -mno-tbm -mno-xop -mno-lwp else $(error gcc or clang are required for this configuration.) endif diff --git a/frame/3/syrk/bli_syrk_front.c b/frame/3/syrk/bli_syrk_front.c index 534848e33..cc2163fae 100644 --- a/frame/3/syrk/bli_syrk_front.c +++ b/frame/3/syrk/bli_syrk_front.c @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -46,7 +47,9 @@ void bli_syrk_front ) { bli_init_once(); - +#ifdef BLIS_ENABLE_SMALL_MATRIX + gint_t status = BLIS_FAILURE; +#endif obj_t a_local; obj_t at_local; obj_t c_local; @@ -68,6 +71,29 @@ void bli_syrk_front bli_obj_set_as_root( &c_local ); // For syrk, the right-hand "B" operand is simply A^T. +#ifdef BLIS_ENABLE_SMALL_MATRIX + bli_obj_alias_to( a, &at_local ); + if (bli_obj_has_trans(a) != 0) + {//At*A operation + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &at_local ); + //call small syrk. + //syrk small matrix threshold check is done inside bli_syrk_small(). + status = bli_syrk_small( alpha, &a_local, &at_local, beta, &c_local, cntx, cntl ); + } + else if ((a->dim[0] <= BLIS_SMALL_MATRIX_A_THRES_M_SYRK && a->dim[1] < BLIS_SMALL_MATRIX_A_THRES_N_SYRK) || + (a->dim[0] < BLIS_SMALL_MATRIX_A_THRES_M_SYRK && a->dim[1] <= BLIS_SMALL_MATRIX_A_THRES_N_SYRK)) + {//A*At operation + bli_obj_set_conjtrans( BLIS_TRANSPOSE, &at_local ); + //call small syrk. + //Explicit matrix dimension threshold check in this else if section before calling bli_syrk_small(). + status = bli_syrk_small( alpha, &a_local, &at_local, beta, &c_local, cntx, cntl ); + } + if ( status == BLIS_SUCCESS ) + { + return; + } +#endif + bli_obj_alias_to( a, &at_local ); bli_obj_induce_trans( &at_local ); diff --git a/frame/3/syrk/bli_syrk_front.h b/frame/3/syrk/bli_syrk_front.h index 28d1e13f6..98b1e1251 100644 --- a/frame/3/syrk/bli_syrk_front.h +++ b/frame/3/syrk/bli_syrk_front.h @@ -42,3 +42,14 @@ void bli_syrk_front rntm_t* rntm, cntl_t* cntl ); + +err_t bli_syrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); \ No newline at end of file diff --git a/frame/3/trsm/bli_trsm_front.c b/frame/3/trsm/bli_trsm_front.c index 5093d1a4a..303570179 100644 --- a/frame/3/trsm/bli_trsm_front.c +++ b/frame/3/trsm/bli_trsm_front.c @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -34,6 +35,8 @@ #include "blis.h" +//#define PRINT_SMALL_TRSM_INFO + void bli_trsm_front ( side_t side, @@ -50,6 +53,47 @@ void bli_trsm_front obj_t a_local; obj_t b_local; obj_t c_local; + + +#ifdef PRINT_SMALL_TRSM_INFO + printf("Side:: %c\n", side ? 'R' : 'L'); + if (bli_obj_datatype(*a) == BLIS_FLOAT) + printf("Alpha:: %9.2e\n", *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, *alpha))); + else if (bli_obj_datatype(*a) == BLIS_DOUBLE) + printf("Alpha is double:: %9.2e\n", *((double *)bli_obj_buffer_for_const(BLIS_DOUBLE, *alpha))); + else + printf("Unsupported datatype for Alpha\n"); + + printf("A:: M = %d, N = %d, elem_size = %d, row_off = %ld, col_off = %ld, rs = %d, cs = %d, trans = %c, TRIANG = %c, unit diag = %c\n", a->dim[0], a->dim[1], bli_obj_elem_size(*a ), bli_obj_row_off(*a), bli_obj_col_off(*a), a->rs, a->cs, bli_obj_has_trans(*a) ? 'Y' : 'N', bli_obj_is_upper(*a) ? 'U' : bli_obj_is_lower(*a) ? 'L' : 'N', bli_obj_has_unit_diag(*a) ? 'Y' : 'N'); +#ifdef PRINT_SMALL_TRSM + //bli_printm("a", a, "%4.1f", ""); +#endif + printf("B:: M = %d, N = %d, elem_size = %d, row_off = %ld, col_off = %ld, rs = %d, cs = %d, trans = %c\n", b->dim[0], b->dim[1], bli_obj_elem_size(*a ), bli_obj_row_off(*a), bli_obj_col_off(*a), b->rs, b->cs, bli_obj_has_trans(*b) ? 'Y' : 'N'); +#ifdef PRINT_SMALL_TRSM + //bli_printm("b", b, "%4.1f", ""); +#endif + fflush(stdout); +#endif +#if 0 +for (i = 0; i < m; i++) //no. of cols of B +{ + for (j = 0; j < n; j++) //no. of rows of B + { + B[i*n + j] = 1001 + j + (i*n); + } +} +for (i = 0; i < m; i++) //no. of cols of B +{ + for (j = i; j < m; j++) //no. of rows of B + { + L[i*m + j] = 2001 + j + (i*m); + } +} +#endif +#ifdef BLIS_ENABLE_SMALL_MATRIX_TRSM + gint_t status = bli_trsm_small( side, alpha, a, b, cntx, cntl ); + if ( status == BLIS_SUCCESS ) return; +#endif // Check parameters. if ( bli_error_checking_is_enabled() ) diff --git a/frame/3/trsm/bli_trsm_front.h b/frame/3/trsm/bli_trsm_front.h index 1a08b7c75..cd65e4454 100644 --- a/frame/3/trsm/bli_trsm_front.h +++ b/frame/3/trsm/bli_trsm_front.h @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018, Advanced Micro Devices, Inc Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -42,3 +43,13 @@ void bli_trsm_front rntm_t* rntm, cntl_t* cntl ); + +err_t bli_trsm_small + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); diff --git a/kernels/zen/1/bli_amaxv_zen_int.c b/kernels/zen/1/bli_amaxv_zen_int.c index aa1aa0e66..ccf6919cc 100644 --- a/kernels/zen/1/bli_amaxv_zen_int.c +++ b/kernels/zen/1/bli_amaxv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without @@ -219,6 +219,12 @@ void bli_samaxv_zen_int } } + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // later, especially if BLIS is compiled with -mfpmath=sse). + _mm256_zeroupper(); + /* Store final index to output variable. */ *i_max = i_max_l; } @@ -370,6 +376,12 @@ void bli_damaxv_zen_int } } + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // later, especially if BLIS is compiled with -mfpmath=sse). + _mm256_zeroupper(); + /* Store final index to output variable. */ *i_max = i_max_l; } diff --git a/kernels/zen/1/bli_axpyv_zen_int.c b/kernels/zen/1/bli_axpyv_zen_int.c index 42668a0a7..bd7cec06a 100644 --- a/kernels/zen/1/bli_axpyv_zen_int.c +++ b/kernels/zen/1/bli_axpyv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without @@ -136,6 +136,13 @@ void bli_saxpyv_zen_int y0 += n_elem_per_reg * n_iter_unroll; } + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + const float alphac = *alpha; // If there are leftover iterations, perform them with scalar code. @@ -233,6 +240,13 @@ void bli_daxpyv_zen_int y0 += n_elem_per_reg * n_iter_unroll; } + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + const double alphac = *alpha; // If there are leftover iterations, perform them with scalar code. diff --git a/kernels/zen/1/bli_axpyv_zen_int10.c b/kernels/zen/1/bli_axpyv_zen_int10.c index d2780d39c..cacbcc6fb 100644 --- a/kernels/zen/1/bli_axpyv_zen_int10.c +++ b/kernels/zen/1/bli_axpyv_zen_int10.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without @@ -228,6 +228,13 @@ void bli_saxpyv_zen_int10 y0 += 1*n_elem_per_reg; } + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + for ( ; (i + 0) < n; i += 1 ) { *y0 += (*alpha) * (*x0); @@ -427,6 +434,13 @@ void bli_daxpyv_zen_int10 y0 += 1*n_elem_per_reg; } + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + for ( ; i < n; i += 1 ) { *y0 += (*alpha) * (*x0); diff --git a/kernels/zen/1/bli_dotv_zen_int.c b/kernels/zen/1/bli_dotv_zen_int.c index 1c87a0f87..32b43bd10 100644 --- a/kernels/zen/1/bli_dotv_zen_int.c +++ b/kernels/zen/1/bli_dotv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without @@ -151,6 +151,13 @@ void bli_sdotv_zen_int rho0 += rho0v.f[0] + rho0v.f[1] + rho0v.f[2] + rho0v.f[3] + rho0v.f[4] + rho0v.f[5] + rho0v.f[6] + rho0v.f[7]; + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + // If there are leftover iterations, perform them with scalar code. for ( i = 0; i < n_left; ++i ) { @@ -265,6 +272,13 @@ void bli_ddotv_zen_int // Accumulate the final rho vector into a single scalar result. rho0 += rho0v.d[0] + rho0v.d[1] + rho0v.d[2] + rho0v.d[3]; + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + // If there are leftover iterations, perform them with scalar code. for ( i = 0; i < n_left; ++i ) { diff --git a/kernels/zen/1/bli_dotv_zen_int10.c b/kernels/zen/1/bli_dotv_zen_int10.c index 79fdde969..8874303af 100644 --- a/kernels/zen/1/bli_dotv_zen_int10.c +++ b/kernels/zen/1/bli_dotv_zen_int10.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without @@ -224,6 +224,13 @@ void bli_sdotv_zen_int10 // Manually add the results from above to finish the sum. rho0 += rhov[0].f[0] + rhov[0].f[4]; rho0 += rhov[1].f[0] + rhov[1].f[4]; + + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // later, especially if BLIS is compiled with -mfpmath=sse). + _mm256_zeroupper(); + } else { @@ -407,6 +414,12 @@ void bli_ddotv_zen_int10 // Manually add the results from above to finish the sum. rho0 += rhov[0].d[0] + rhov[0].d[1] + rhov[0].d[2] + rhov[0].d[3]; rho0 += rhov[1].d[0] + rhov[1].d[1] + rhov[1].d[2] + rhov[1].d[3]; + + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // later, especially if BLIS is compiled with -mfpmath=sse). + _mm256_zeroupper(); } else { diff --git a/kernels/zen/1/bli_dotxv_zen_int.c b/kernels/zen/1/bli_dotxv_zen_int.c index 53b582b77..d27225934 100644 --- a/kernels/zen/1/bli_dotxv_zen_int.c +++ b/kernels/zen/1/bli_dotxv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without @@ -157,6 +157,13 @@ void bli_sdotxv_zen_int rho0 = rho0v.f[0] + rho0v.f[1] + rho0v.f[2] + rho0v.f[3] + rho0v.f[4] + rho0v.f[5] + rho0v.f[6] + rho0v.f[7]; + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + // If there are leftover iterations, perform them with scalar code. for ( i = 0; i < n_left; ++i ) { @@ -277,6 +284,13 @@ void bli_ddotxv_zen_int // Accumulate the final rho vector into a single scalar result. rho0 = rho0v.d[0] + rho0v.d[1] + rho0v.d[2] + rho0v.d[3]; + // Issue vzeroupper instruction to clear upper lanes of ymm registers. + // This avoids a performance penalty caused by false dependencies when + // transitioning from from AVX to SSE instructions (which may occur + // as soon as the n_left cleanup loop below if BLIS is compiled with + // -mfpmath=sse). + _mm256_zeroupper(); + // If there are leftover iterations, perform them with scalar code. for ( i = 0; i < n_left; ++i ) { diff --git a/kernels/zen/3/bli_syrk_small.c b/kernels/zen/3/bli_syrk_small.c new file mode 100644 index 000000000..d10114b40 --- /dev/null +++ b/kernels/zen/3/bli_syrk_small.c @@ -0,0 +1,4186 @@ +/* + +BLIS +An object-based framework for developing high-performance BLAS-like +libraries. + +Copyright (C) 2018, Advanced Micro Devices, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of The University of Texas at Austin nor the names +of its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "immintrin.h" +#include "xmmintrin.h" +#include "blis.h" + +#ifdef BLIS_ENABLE_SMALL_MATRIX + +#define MR 32 +#define D_MR (MR >> 1) +#define NR 3 + +#define BLIS_ENABLE_PREFETCH +#define F_SCRATCH_DIM (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES) +static float A_pack[F_SCRATCH_DIM] __attribute__((aligned(64))); +static float C_pack[F_SCRATCH_DIM] __attribute__((aligned(64))); +#define D_BLIS_SMALL_MATRIX_THRES (BLIS_SMALL_MATRIX_THRES / 2 ) +#define D_BLIS_SMALL_M_RECT_MATRIX_THRES (BLIS_SMALL_M_RECT_MATRIX_THRES / 2) +#define D_BLIS_SMALL_K_RECT_MATRIX_THRES (BLIS_SMALL_K_RECT_MATRIX_THRES / 2) +#define D_SCRATCH_DIM (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES) +static double D_A_pack[D_SCRATCH_DIM] __attribute__((aligned(64))); +static double D_C_pack[D_SCRATCH_DIM] __attribute__((aligned(64))); +#define BLIS_ATBN_M_THRES 40 // Threshold value of M for/below which small matrix code is called. +#define AT_MR 4 // The kernel dimension of the A transpose SYRK kernel.(AT_MR * NR). +static err_t bli_ssyrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); + +static err_t bli_dsyrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); + +static err_t bli_ssyrk_small_atbn + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); + +static err_t bli_dsyrk_small_atbn + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ); +/* +* The bli_syrk_small function will use the +* custom MRxNR kernels, to perform the computation. +* The custom kernels are used if the [M * N] < 240 * 240 +*/ +err_t bli_syrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ +#ifdef BLIS_ENABLE_MULTITHREADING + return BLIS_NOT_YET_IMPLEMENTED; +#endif + // If alpha is zero, scale by beta and return. + if (bli_obj_equals(alpha, &BLIS_ZERO)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + // if row major format return. + if ((bli_obj_row_stride( a ) != 1) || + (bli_obj_row_stride( b ) != 1) || + (bli_obj_row_stride( c ) != 1)) + { + return BLIS_INVALID_ROW_STRIDE; + } + + num_t dt = ((*c).info & (0x7 << 0)); + + if (bli_obj_has_trans( a )) + { + if (bli_obj_has_notrans( b )) + { + if (dt == BLIS_FLOAT) + { + return bli_ssyrk_small_atbn(alpha, a, b, beta, c, cntx, cntl); + } + else if (dt == BLIS_DOUBLE) + { + return bli_dsyrk_small_atbn(alpha, a, b, beta, c, cntx, cntl); + } + } + + return BLIS_NOT_YET_IMPLEMENTED; + } + + if (dt == BLIS_DOUBLE) + { + return bli_dsyrk_small(alpha, a, b, beta, c, cntx, cntl); + } + + if (dt == BLIS_FLOAT) + { + return bli_ssyrk_small(alpha, a, b, beta, c, cntx, cntl); + } + + return BLIS_NOT_YET_IMPLEMENTED; +}; + + +static err_t bli_ssyrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + int M = bli_obj_length( c ); // number of rows of Matrix C + int N = bli_obj_width( c ); // number of columns of Matrix C + int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + int L = M * N; + + if ((((L) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES)) + || ((M < BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) + { + + int lda = bli_obj_col_stride(a); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + int ldb = bli_obj_col_stride(b); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + int ldc_matC = bli_obj_col_stride( c ); // column stride of matrix C + int ldc = M;//bli_obj_col_stride( c ); // column stride of static buffer for matrix C + int row_idx, col_idx, k; + int rs_matC = bli_obj_row_stride( c ); + int rsc = 1; + float *A = a->buffer; // pointer to elements of Matrix A + float *B = b->buffer; // pointer to elements of Matrix B + float *C = C_pack; // pointer to elements of Matrix C + float *matCbuf = c->buffer; + + float *tA = A, *tB = B, *tC = C;//, *tA_pack; + float *tA_packed; // temprorary pointer to hold packed A memory pointer + int row_idx_packed; //packed A memory row index + int lda_packed; //lda of packed A + int col_idx_start; //starting index after A matrix is packed. + dim_t tb_inc_row = 1; // row stride of matrix B + dim_t tb_inc_col = ldb; // column stride of matrix B + __m256 ymm4, ymm5, ymm6, ymm7; + __m256 ymm8, ymm9, ymm10, ymm11; + __m256 ymm12, ymm13, ymm14, ymm15; + __m256 ymm0, ymm1, ymm2, ymm3; + + int n_remainder; // If the N is non multiple of 3.(N%3) + int m_remainder; // If the M is non multiple of 32.(M%32) + + float *alpha_cast, *beta_cast; // alpha, beta multiples + alpha_cast = (alpha->buffer); + beta_cast = (beta->buffer); + int required_packing_A = 1; + + // when N is equal to 1 call GEMV instead of SYRK + if (N == 1) + { + bli_gemv + ( + alpha, + a, + b, + beta, + c + ); + return BLIS_SUCCESS; + } + + //update the pointer math if matrix B needs to be transposed. + if (bli_obj_has_trans( b )) + { + tb_inc_col = 1; //switch row and column strides + tb_inc_row = ldb; + } + + if ((N <= 3) || ((MR * K) > F_SCRATCH_DIM)) + { + required_packing_A = 0; + } + /* + * The computation loop runs for MRxN columns of C matrix, thus + * accessing the MRxK A matrix data and KxNR B matrix data. + * The computation is organized as inner loops of dimension MRxNR. + */ + // Process MR rows of C matrix at a time. + for (row_idx = 0; (row_idx + (MR - 1)) < M; row_idx += MR) + { + + col_idx_start = 0; + tA_packed = A; + row_idx_packed = row_idx; + lda_packed = lda; + + // This is the part of the pack and compute optimization. + // During the first column iteration, we store the accessed A matrix into + // contiguous static memory. This helps to keep te A matrix in Cache and + // aviods the TLB misses. + if (required_packing_A) + { + col_idx = 0; + + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + tA_packed = A_pack; + +#if 0//def BLIS_ENABLE_PREFETCH + _mm_prefetch((char*)(tC + 0), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 16), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc + 16), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc + 16), _MM_HINT_T0); +#endif + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm11 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + // This loop is processing MR x K + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + _mm256_storeu_ps(tA_packed, ymm3); // the packing of matrix A + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + _mm256_storeu_ps(tA_packed + 8, ymm3); // the packing of matrix A + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_ps(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_ps(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + _mm256_storeu_ps(tA_packed + 16, ymm3); // the packing of matrix A + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_ps(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_ps(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_ps(ymm2, ymm3, ymm14); + + ymm3 = _mm256_loadu_ps(tA + 24); + _mm256_storeu_ps(tA_packed + 24, ymm3); // the packing of matrix A + // ymm7 += ymm0 * ymm3; + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + // ymm11 += ymm1 * ymm3; + ymm11 = _mm256_fmadd_ps(ymm1, ymm3, ymm11); + // ymm15 += ymm2 * ymm3; + ymm15 = _mm256_fmadd_ps(ymm2, ymm3, ymm15); + + tA += lda; + tA_packed += MR; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm11 = _mm256_mul_ps(ymm11, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + ymm15 = _mm256_mul_ps(ymm15, ymm0); + + // multiply C by beta and accumulate col 1. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + _mm256_storeu_ps(tC + 16, ymm6); + _mm256_storeu_ps(tC + 24, ymm7); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm11 = _mm256_fmadd_ps(ymm2, ymm1, ymm11);*/ + _mm256_storeu_ps(tC, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + _mm256_storeu_ps(tC + 24, ymm11); + + // multiply C by beta and accumulate, col 3. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm15 = _mm256_fmadd_ps(ymm2, ymm1, ymm15);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + _mm256_storeu_ps(tC + 24, ymm15); + + // modify the pointer arithematic to use packed A matrix. + col_idx_start = NR; + tA_packed = A_pack; + row_idx_packed = 0; + lda_packed = MR; + } + // Process NR columns of C matrix at a time. + for (col_idx = col_idx_start; (col_idx + (NR - 1)) < N; col_idx += NR) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = tA_packed + row_idx_packed; + +#if 0//def BLIS_ENABLE_PREFETCH + _mm_prefetch((char*)(tC + 0), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 16), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc + 16), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc + 16), _MM_HINT_T0); +#endif + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm11 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + // This loop is processing MR x K + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_ps(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_ps(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_ps(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_ps(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_ps(ymm2, ymm3, ymm14); + + ymm3 = _mm256_loadu_ps(tA + 24); + // ymm7 += ymm0 * ymm3; + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + // ymm11 += ymm1 * ymm3; + ymm11 = _mm256_fmadd_ps(ymm1, ymm3, ymm11); + // ymm15 += ymm2 * ymm3; + ymm15 = _mm256_fmadd_ps(ymm2, ymm3, ymm15); + + tA += lda_packed; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm11 = _mm256_mul_ps(ymm11, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + ymm15 = _mm256_mul_ps(ymm15, ymm0); + + // multiply C by beta and accumulate col 1. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + _mm256_storeu_ps(tC + 16, ymm6); + _mm256_storeu_ps(tC + 24, ymm7); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm11 = _mm256_fmadd_ps(ymm2, ymm1, ymm11);*/ + _mm256_storeu_ps(tC, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + _mm256_storeu_ps(tC + 24, ymm11); + + // multiply C by beta and accumulate, col 3. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm15 = _mm256_fmadd_ps(ymm2, ymm1, ymm15);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + _mm256_storeu_ps(tC + 24, ymm15); + + } + n_remainder = N - col_idx; + + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm11 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm8 = _mm256_fmadd_ps(ymm0, ymm3, ymm8); + ymm12 = _mm256_fmadd_ps(ymm1, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm9 = _mm256_fmadd_ps(ymm0, ymm3, ymm9); + ymm13 = _mm256_fmadd_ps(ymm1, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + ymm14 = _mm256_fmadd_ps(ymm1, ymm3, ymm14); + + ymm3 = _mm256_loadu_ps(tA + 24); + ymm11 = _mm256_fmadd_ps(ymm0, ymm3, ymm11); + ymm15 = _mm256_fmadd_ps(ymm1, ymm3, ymm15); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm11 = _mm256_mul_ps(ymm11, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + ymm15 = _mm256_mul_ps(ymm15, ymm0); + + // multiply C by beta and accumulate, col 1. + /*ymm2 = _mm256_loadu_ps(tC + 0); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm11 = _mm256_fmadd_ps(ymm2, ymm1, ymm11);*/ + _mm256_storeu_ps(tC + 0, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + _mm256_storeu_ps(tC + 24, ymm11); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm15 = _mm256_fmadd_ps(ymm2, ymm1, ymm15);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + _mm256_storeu_ps(tC + 24, ymm15); + + col_idx += 2; + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm12 = _mm256_fmadd_ps(ymm0, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + ymm14 = _mm256_fmadd_ps(ymm0, ymm3, ymm14); + + ymm3 = _mm256_loadu_ps(tA + 24); + ymm15 = _mm256_fmadd_ps(ymm0, ymm3, ymm15); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + ymm15 = _mm256_mul_ps(ymm15, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC + 0); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_ps(tC + 24); + ymm15 = _mm256_fmadd_ps(ymm2, ymm1, ymm15);*/ + + _mm256_storeu_ps(tC + 0, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + _mm256_storeu_ps(tC + 24, ymm15); + } + } + + m_remainder = M - row_idx; + + if (m_remainder >= 24) + { + m_remainder -= 24; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_ps(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_ps(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_ps(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_ps(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_ps(ymm2, ymm3, ymm14); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + _mm256_storeu_ps(tC + 16, ymm6); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10);*/ + _mm256_storeu_ps(tC, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm8 = _mm256_fmadd_ps(ymm0, ymm3, ymm8); + ymm12 = _mm256_fmadd_ps(ymm1, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm9 = _mm256_fmadd_ps(ymm0, ymm3, ymm9); + ymm13 = _mm256_fmadd_ps(ymm1, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + ymm14 = _mm256_fmadd_ps(ymm1, ymm3, ymm14); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + ymm10 = _mm256_mul_ps(ymm10, ymm0); + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC + 0); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm10 = _mm256_fmadd_ps(ymm2, ymm1, ymm10);*/ + _mm256_storeu_ps(tC + 0, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + _mm256_storeu_ps(tC + 16, ymm10); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14);*/ + _mm256_storeu_ps(tC, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + + col_idx += 2; + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm12 = _mm256_fmadd_ps(ymm0, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + + ymm3 = _mm256_loadu_ps(tA + 16); + ymm14 = _mm256_fmadd_ps(ymm0, ymm3, ymm14); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm12 = _mm256_mul_ps(ymm12, ymm0); + ymm13 = _mm256_mul_ps(ymm13, ymm0); + ymm14 = _mm256_mul_ps(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC + 0); + ymm12 = _mm256_fmadd_ps(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm13 = _mm256_fmadd_ps(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_ps(tC + 16); + ymm14 = _mm256_fmadd_ps(ymm2, ymm1, ymm14);*/ + + _mm256_storeu_ps(tC + 0, ymm12); + _mm256_storeu_ps(tC + 8, ymm13); + _mm256_storeu_ps(tC + 16, ymm14); + } + + row_idx += 24; + } + + if (m_remainder >= 16) + { + m_remainder -= 16; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm6 = _mm256_fmadd_ps(ymm1, ymm3, ymm6); + ymm8 = _mm256_fmadd_ps(ymm2, ymm3, ymm8); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + ymm8 = _mm256_mul_ps(ymm8, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(tC, ymm6); + _mm256_storeu_ps(tC + 8, ymm7); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm8 = _mm256_fmadd_ps(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9);*/ + _mm256_storeu_ps(tC, ymm8); + _mm256_storeu_ps(tC + 8, ymm9); + + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm6 = _mm256_fmadd_ps(ymm1, ymm3, ymm6); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(tC, ymm6); + _mm256_storeu_ps(tC + 8, ymm7); + + col_idx += 2; + + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + ymm3 = _mm256_loadu_ps(tA + 8); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_ps(tC + 8); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm4); + _mm256_storeu_ps(tC + 8, ymm5); + + } + + row_idx += 16; + } + + if (m_remainder >= 8) + { + m_remainder -= 8; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_ps(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_ps(ymm2, ymm3, ymm6); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm6 = _mm256_mul_ps(ymm6, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4);*/ + _mm256_storeu_ps(tC, ymm4); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm6 = _mm256_fmadd_ps(ymm2, ymm1, ymm6);*/ + _mm256_storeu_ps(tC, ymm6); + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_ps(ymm1, ymm3, ymm5); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_ps(ymm4, ymm0); + ymm5 = _mm256_mul_ps(ymm5, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4);*/ + _mm256_storeu_ps(tC, ymm4); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_ps(tC); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(tC, ymm5); + + col_idx += 2; + + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_ps(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + ymm4 = _mm256_mul_ps(ymm4, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_ps(tC); + ymm4 = _mm256_fmadd_ps(ymm2, ymm1, ymm4);*/ + _mm256_storeu_ps(tC, ymm4); + + } + + row_idx += 8; + } + // M is not a multiple of 32. + // The handling of edge case where the remainder + // dimension is less than 8. The padding takes place + // to handle this case. + if ((m_remainder) && (lda > 7)) + { + float f_temp[8]; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm5 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_ps(tA); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_ss(tB + tb_inc_col * 2); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + //multiply A*B by alpha. + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + ymm9 = _mm256_mul_ps(ymm9, ymm0); + + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(f_temp, ymm7); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm9 = _mm256_fmadd_ps(ymm2, ymm1, ymm9);*/ + _mm256_storeu_ps(f_temp, ymm9); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm5 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + ymm3 = _mm256_loadu_ps(tA); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + + tA += lda; + } + + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_ss(tB + tb_inc_col * 1); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_ps(ymm1, ymm3, ymm7); + + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + ymm5 = _mm256_mul_ps(ymm5, ymm0); + ymm7 = _mm256_mul_ps(ymm7, ymm0); + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm7 = _mm256_fmadd_ps(ymm2, ymm1, ymm7);*/ + _mm256_storeu_ps(f_temp, ymm7); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm5 = _mm256_setzero_ps(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + ymm3 = _mm256_loadu_ps(tA); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + + tA += lda; + } + + ymm0 = _mm256_broadcast_ss(tB + tb_inc_col * 0); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm0, ymm3, ymm5); + + ymm0 = _mm256_broadcast_ss(alpha_cast); + //ymm1 = _mm256_broadcast_ss(beta_cast); + + // multiply C by beta and accumulate. + ymm5 = _mm256_mul_ps(ymm5, ymm0); + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_ps(f_temp); + ymm5 = _mm256_fmadd_ps(ymm2, ymm1, ymm5);*/ + _mm256_storeu_ps(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + m_remainder = 0; + } + + if (m_remainder) + { + float result; + for (; row_idx < M; row_idx += 1) + { + for (col_idx = 0; col_idx < N; col_idx += 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + result = 0; + for (k = 0; k < K; ++k) + { + result += (*tA) * (*tB); + tA += lda; + tB += tb_inc_row; + } + + result *= (*alpha_cast); + (*tC) = /*(*tC) * (*beta_cast) + */result; + } + } + } + + //copy/compute sryk values back to C using SIMD + if ( bli_seq0( *beta_cast ) ) + {//just copy in case of beta = 0 + dim_t _i, _j, k, _l; + if(bli_obj_is_lower(c)) // c is lower + { + //first column + _j = 0; + k = M >> 3; + _i = 0; + for ( _l = 0; _l < k; _l++ ) + { + ymm0 = _mm256_loadu_ps((C + _i*rsc)); + _mm256_storeu_ps((matCbuf + _i*rs_matC), ymm0); + _i += 8; + } + while (_i < M ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + while ( _j < N ) //next column + { + //k = (_j + (8 - (_j & 7))); + _l = _j & 7; + k = (_l != 0) ? (_j + (8 - _l)) : _j; + k = (k <= M) ? k : M; + for ( _i = _j; _i < k; ++_i ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + k = (M - _i) >> 3; + _l = 0; + while ( _l < k ) + { + ymm0 = _mm256_loadu_ps((C + _i*rsc + _j*ldc)); + _mm256_storeu_ps((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + + _i += 8; + _l++; + } + while (_i < M ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + { + k = (_j + 1) >> 3; + _i = 0; + _l = 0; + while ( _l < k ) + { + ymm0 = _mm256_loadu_ps((C + _i*rsc + _j*ldc)); + _mm256_storeu_ps((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + _i += 8; + _l++; + } + while (_i <= _j ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + ++_i; + } + } + } + } + else + {//when beta is non-zero, fmadd and store the results + dim_t _i, _j, k, _l; + ymm1 = _mm256_broadcast_ss(beta_cast); + if(bli_obj_is_lower(c)) //c is lower + { + //first column + _j = 0; + k = M >> 3; + _i = 0; + for ( _l = 0; _l < k; _l++ ) + { + ymm2 = _mm256_loadu_ps((matCbuf + _i*rs_matC)); + ymm0 = _mm256_loadu_ps((C + _i*rsc)); + ymm0 = _mm256_fmadd_ps(ymm2, ymm1, ymm0); + _mm256_storeu_ps((matCbuf + _i*rs_matC), ymm0); + _i += 8; + } + while (_i < M ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + while ( _j < N ) //next column + { + //k = (_j + (8 - (_j & 7))); + _l = _j & 7; + k = (_l != 0) ? (_j + (8 - _l)) : _j; + k = (k <= M) ? k : M; + for ( _i = _j; _i < k; ++_i ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + k = (M - _i) >> 3; + _l = 0; + while ( _l < k ) + { + ymm2 = _mm256_loadu_ps((matCbuf + _i*rs_matC + _j*ldc_matC)); + ymm0 = _mm256_loadu_ps((C + _i*rsc + _j*ldc)); + ymm0 = _mm256_fmadd_ps(ymm2, ymm1, ymm0); + _mm256_storeu_ps((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + + _i += 8; + _l++; + } + while (_i < M ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + { + k = (_j + 1) >> 3; + _i = 0; + _l = 0; + while ( _l < k ) + { + ymm2 = _mm256_loadu_ps((matCbuf + _i*rs_matC + _j*ldc_matC)); + ymm0 = _mm256_loadu_ps((C + _i*rsc + _j*ldc)); + ymm0 = _mm256_fmadd_ps(ymm2, ymm1, ymm0); + _mm256_storeu_ps((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + _i += 8; + _l++; + } + while (_i <= _j ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + ++_i; + } + } + } + } + + return BLIS_SUCCESS; + } + else + return BLIS_NONCONFORMAL_DIMENSIONS; + + +}; + +static err_t bli_dsyrk_small + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + int M = bli_obj_length( c ); // number of rows of Matrix C + int N = bli_obj_width( c ); // number of columns of Matrix C + int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + int L = M * N; + + // If alpha is zero, scale by beta and return. + if ((((L) < (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES)) + || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) + { + + int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + int ldc_matC = bli_obj_col_stride( c ); // column stride of matrix C + int ldc = M;//bli_obj_col_stride( c ); // column stride of static buffer for matrix C + int row_idx, col_idx, k; + int rs_matC = bli_obj_row_stride( c ); + int rsc = 1; + double *A = a->buffer; // pointer to elements of Matrix A + double *B = b->buffer; // pointer to elements of Matrix B + double *C = D_C_pack; // pointer to elements of Matrix C + double *matCbuf = c->buffer; + + double *tA = A, *tB = B, *tC = C;//, *tA_pack; + double *tA_packed; // temprorary pointer to hold packed A memory pointer + int row_idx_packed; //packed A memory row index + int lda_packed; //lda of packed A + int col_idx_start; //starting index after A matrix is packed. + dim_t tb_inc_row = 1; // row stride of matrix B + dim_t tb_inc_col = ldb; // column stride of matrix B + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm0, ymm1, ymm2, ymm3; + + int n_remainder; // If the N is non multiple of 3.(N%3) + int m_remainder; // If the M is non multiple of 16.(M%16) + + double *alpha_cast, *beta_cast; // alpha, beta multiples + alpha_cast = (alpha->buffer); + beta_cast = (beta->buffer); + int required_packing_A = 1; + + // when N is equal to 1 call GEMV instead of SYRK + if (N == 1) + { + bli_gemv + ( + alpha, + a, + b, + beta, + c + ); + return BLIS_SUCCESS; + } + + //update the pointer math if matrix B needs to be transposed. + if (bli_obj_has_trans( b )) + { + tb_inc_col = 1; //switch row and column strides + tb_inc_row = ldb; + } + + if ((N <= 3) || ((D_MR * K) > D_SCRATCH_DIM)) + { + required_packing_A = 0; + } + /* + * The computation loop runs for D_MRxN columns of C matrix, thus + * accessing the D_MRxK A matrix data and KxNR B matrix data. + * The computation is organized as inner loops of dimension D_MRxNR. + */ + // Process D_MR rows of C matrix at a time. + for (row_idx = 0; (row_idx + (D_MR - 1)) < M; row_idx += D_MR) + { + + col_idx_start = 0; + tA_packed = A; + row_idx_packed = row_idx; + lda_packed = lda; + + // This is the part of the pack and compute optimization. + // During the first column iteration, we store the accessed A matrix into + // contiguous static memory. This helps to keep te A matrix in Cache and + // aviods the TLB misses. + if (required_packing_A) + { + col_idx = 0; + + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + tA_packed = D_A_pack; + +#if 0//def BLIS_ENABLE_PREFETCH + _mm_prefetch((char*)(tC + 0), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 8), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc + 8), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc + 8), _MM_HINT_T0); +#endif + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + // This loop is processing D_MR x K + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + _mm256_storeu_pd(tA_packed, ymm3); // the packing of matrix A + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + _mm256_storeu_pd(tA_packed + 4, ymm3); // the packing of matrix A + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_pd(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_pd(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + _mm256_storeu_pd(tA_packed + 8, ymm3); // the packing of matrix A + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_pd(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_pd(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_pd(ymm2, ymm3, ymm14); + + ymm3 = _mm256_loadu_pd(tA + 12); + _mm256_storeu_pd(tA_packed + 12, ymm3); // the packing of matrix A + // ymm7 += ymm0 * ymm3; + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + // ymm11 += ymm1 * ymm3; + ymm11 = _mm256_fmadd_pd(ymm1, ymm3, ymm11); + // ymm15 += ymm2 * ymm3; + ymm15 = _mm256_fmadd_pd(ymm2, ymm3, ymm15); + + tA += lda; + tA_packed += D_MR; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm11 = _mm256_mul_pd(ymm11, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + // multiply C by beta and accumulate col 1. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + _mm256_storeu_pd(tC + 8, ymm6); + _mm256_storeu_pd(tC + 12, ymm7); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm11 = _mm256_fmadd_pd(ymm2, ymm1, ymm11);*/ + _mm256_storeu_pd(tC, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + _mm256_storeu_pd(tC + 12, ymm11); + + // multiply C by beta and accumulate, col 3. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm15 = _mm256_fmadd_pd(ymm2, ymm1, ymm15);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + _mm256_storeu_pd(tC + 12, ymm15); + + // modify the pointer arithematic to use packed A matrix. + col_idx_start = NR; + tA_packed = D_A_pack; + row_idx_packed = 0; + lda_packed = D_MR; + } + // Process NR columns of C matrix at a time. + for (col_idx = col_idx_start; (col_idx + (NR - 1)) < N; col_idx += NR) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = tA_packed + row_idx_packed; + +#if 0//def BLIS_ENABLE_PREFETCH + _mm_prefetch((char*)(tC + 0), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 8), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + ldc + 8), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc), _MM_HINT_T0); + _mm_prefetch((char*)(tC + 2 * ldc + 8), _MM_HINT_T0); +#endif + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + // This loop is processing D_MR x K + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_pd(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_pd(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_pd(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_pd(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_pd(ymm2, ymm3, ymm14); + + ymm3 = _mm256_loadu_pd(tA + 12); + // ymm7 += ymm0 * ymm3; + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + // ymm11 += ymm1 * ymm3; + ymm11 = _mm256_fmadd_pd(ymm1, ymm3, ymm11); + // ymm15 += ymm2 * ymm3; + ymm15 = _mm256_fmadd_pd(ymm2, ymm3, ymm15); + + tA += lda_packed; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm11 = _mm256_mul_pd(ymm11, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + // multiply C by beta and accumulate col 1. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + _mm256_storeu_pd(tC + 8, ymm6); + _mm256_storeu_pd(tC + 12, ymm7); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm11 = _mm256_fmadd_pd(ymm2, ymm1, ymm11);*/ + _mm256_storeu_pd(tC, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + _mm256_storeu_pd(tC + 12, ymm11); + + // multiply C by beta and accumulate, col 3. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm15 = _mm256_fmadd_pd(ymm2, ymm1, ymm15);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + _mm256_storeu_pd(tC + 12, ymm15); + + } + n_remainder = N - col_idx; + + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm8 = _mm256_fmadd_pd(ymm0, ymm3, ymm8); + ymm12 = _mm256_fmadd_pd(ymm1, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm9 = _mm256_fmadd_pd(ymm0, ymm3, ymm9); + ymm13 = _mm256_fmadd_pd(ymm1, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + ymm14 = _mm256_fmadd_pd(ymm1, ymm3, ymm14); + + ymm3 = _mm256_loadu_pd(tA + 12); + ymm11 = _mm256_fmadd_pd(ymm0, ymm3, ymm11); + ymm15 = _mm256_fmadd_pd(ymm1, ymm3, ymm15); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm11 = _mm256_mul_pd(ymm11, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + // multiply C by beta and accumulate, col 1. + /*ymm2 = _mm256_loadu_pd(tC + 0); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm11 = _mm256_fmadd_pd(ymm2, ymm1, ymm11);*/ + _mm256_storeu_pd(tC + 0, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + _mm256_storeu_pd(tC + 12, ymm11); + + // multiply C by beta and accumulate, col 2. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm15 = _mm256_fmadd_pd(ymm2, ymm1, ymm15);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + _mm256_storeu_pd(tC + 12, ymm15); + + col_idx += 2; + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm12 = _mm256_fmadd_pd(ymm0, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + ymm14 = _mm256_fmadd_pd(ymm0, ymm3, ymm14); + + ymm3 = _mm256_loadu_pd(tA + 12); + ymm15 = _mm256_fmadd_pd(ymm0, ymm3, ymm15); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC + 0); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14); + ymm2 = _mm256_loadu_pd(tC + 12); + ymm15 = _mm256_fmadd_pd(ymm2, ymm1, ymm15);*/ + + _mm256_storeu_pd(tC + 0, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + _mm256_storeu_pd(tC + 12, ymm15); + } + } + + m_remainder = M - row_idx; + + if (m_remainder >= 12) + { + m_remainder -= 12; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + // ymm4 += ymm0 * ymm3; + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + // ymm8 += ymm1 * ymm3; + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + // ymm12 += ymm2 * ymm3; + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + // ymm5 += ymm0 * ymm3; + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + // ymm9 += ymm1 * ymm3; + ymm9 = _mm256_fmadd_pd(ymm1, ymm3, ymm9); + // ymm13 += ymm2 * ymm3; + ymm13 = _mm256_fmadd_pd(ymm2, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + // ymm6 += ymm0 * ymm3; + ymm6 = _mm256_fmadd_pd(ymm0, ymm3, ymm6); + // ymm10 += ymm1 * ymm3; + ymm10 = _mm256_fmadd_pd(ymm1, ymm3, ymm10); + // ymm14 += ymm2 * ymm3; + ymm14 = _mm256_fmadd_pd(ymm2, ymm3, ymm14); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + _mm256_storeu_pd(tC + 8, ymm6); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10);*/ + _mm256_storeu_pd(tC, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm8 = _mm256_fmadd_pd(ymm0, ymm3, ymm8); + ymm12 = _mm256_fmadd_pd(ymm1, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm9 = _mm256_fmadd_pd(ymm0, ymm3, ymm9); + ymm13 = _mm256_fmadd_pd(ymm1, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + ymm14 = _mm256_fmadd_pd(ymm1, ymm3, ymm14); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + ymm10 = _mm256_mul_pd(ymm10, ymm0); + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC + 0); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm10 = _mm256_fmadd_pd(ymm2, ymm1, ymm10);*/ + _mm256_storeu_pd(tC + 0, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + _mm256_storeu_pd(tC + 8, ymm10); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14);*/ + _mm256_storeu_pd(tC, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + + col_idx += 2; + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm12 = _mm256_fmadd_pd(ymm0, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + + ymm3 = _mm256_loadu_pd(tA + 8); + ymm14 = _mm256_fmadd_pd(ymm0, ymm3, ymm14); + + tA += lda; + + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm12 = _mm256_mul_pd(ymm12, ymm0); + ymm13 = _mm256_mul_pd(ymm13, ymm0); + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC + 0); + ymm12 = _mm256_fmadd_pd(ymm2, ymm1, ymm12); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm13 = _mm256_fmadd_pd(ymm2, ymm1, ymm13); + ymm2 = _mm256_loadu_pd(tC + 8); + ymm14 = _mm256_fmadd_pd(ymm2, ymm1, ymm14);*/ + + _mm256_storeu_pd(tC + 0, ymm12); + _mm256_storeu_pd(tC + 4, ymm13); + _mm256_storeu_pd(tC + 8, ymm14); + } + + row_idx += 12; + } + + if (m_remainder >= 8) + { + m_remainder -= 8; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm6 = _mm256_fmadd_pd(ymm1, ymm3, ymm6); + ymm8 = _mm256_fmadd_pd(ymm2, ymm3, ymm8); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + ymm8 = _mm256_mul_pd(ymm8, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(tC, ymm6); + _mm256_storeu_pd(tC + 4, ymm7); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm8 = _mm256_fmadd_pd(ymm2, ymm1, ymm8); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9);*/ + _mm256_storeu_pd(tC, ymm8); + _mm256_storeu_pd(tC + 4, ymm9); + + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm6 = _mm256_fmadd_pd(ymm1, ymm3, ymm6); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(tC, ymm6); + _mm256_storeu_pd(tC + 4, ymm7); + + col_idx += 2; + + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + ymm3 = _mm256_loadu_pd(tA + 4); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4); + ymm2 = _mm256_loadu_pd(tC + 4); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm4); + _mm256_storeu_pd(tC + 4, ymm5); + + } + + row_idx += 8; + } + + if (m_remainder >= 4) + { + m_remainder -= 4; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_pd(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_pd(ymm2, ymm3, ymm6); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm6 = _mm256_mul_pd(ymm6, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4);*/ + _mm256_storeu_pd(tC, ymm4); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm5); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm6 = _mm256_fmadd_pd(ymm2, ymm1, ymm6);*/ + _mm256_storeu_pd(tC, ymm6); + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_pd(ymm1, ymm3, ymm5); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm4 = _mm256_mul_pd(ymm4, ymm0); + ymm5 = _mm256_mul_pd(ymm5, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4);*/ + _mm256_storeu_pd(tC, ymm4); + + // multiply C by beta and accumulate. + tC += ldc; + /*ymm2 = _mm256_loadu_pd(tC); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(tC, ymm5); + + col_idx += 2; + + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm4 = _mm256_setzero_pd(); + + for (k = 0; k < K; ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + ymm4 = _mm256_mul_pd(ymm4, ymm0); + + // multiply C by beta and accumulate. + /*ymm2 = _mm256_loadu_pd(tC); + ymm4 = _mm256_fmadd_pd(ymm2, ymm1, ymm4);*/ + _mm256_storeu_pd(tC, ymm4); + + } + + row_idx += 4; + } + // M is not a multiple of 32. + // The handling of edge case where the remainder + // dimension is less than 8. The padding takes place + // to handle this case. + if ((m_remainder) && (lda > 3)) + { + double f_temp[8]; + + for (col_idx = 0; (col_idx + 2) < N; col_idx += 3) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + // clear scratch registers. + ymm5 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + //broadcasted matrix B elements are multiplied + //with matrix A columns. + ymm3 = _mm256_loadu_pd(tA); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + tA += lda; + } + // alpha, beta multiplication. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + ymm2 = _mm256_broadcast_sd(tB + tb_inc_col * 2); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + //multiply A*B by alpha. + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + ymm9 = _mm256_mul_pd(ymm9, ymm0); + + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(f_temp, ymm7); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm9 = _mm256_fmadd_pd(ymm2, ymm1, ymm9);*/ + _mm256_storeu_pd(f_temp, ymm9); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + n_remainder = N - col_idx; + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 2) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm5 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + ymm3 = _mm256_loadu_pd(tA); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + + tA += lda; + } + + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + ymm1 = _mm256_broadcast_sd(tB + tb_inc_col * 1); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + ymm7 = _mm256_fmadd_pd(ymm1, ymm3, ymm7); + + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + ymm5 = _mm256_mul_pd(ymm5, ymm0); + ymm7 = _mm256_mul_pd(ymm7, ymm0); + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + + tC += ldc; + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm7 = _mm256_fmadd_pd(ymm2, ymm1, ymm7);*/ + _mm256_storeu_pd(f_temp, ymm7); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + // if the N is not multiple of 3. + // handling edge case. + if (n_remainder == 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + ymm5 = _mm256_setzero_pd(); + + for (k = 0; k < (K - 1); ++k) + { + // The inner loop broadcasts the B matrix data and + // multiplies it with the A matrix. + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + ymm3 = _mm256_loadu_pd(tA); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + + tA += lda; + } + + ymm0 = _mm256_broadcast_sd(tB + tb_inc_col * 0); + tB += tb_inc_row; + + for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tA[i]; + } + ymm3 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm0, ymm3, ymm5); + + ymm0 = _mm256_broadcast_sd(alpha_cast); + //ymm1 = _mm256_broadcast_sd(beta_cast); + + // multiply C by beta and accumulate. + ymm5 = _mm256_mul_pd(ymm5, ymm0); + + /*for (int i = 0; i < m_remainder; i++) + { + f_temp[i] = tC[i]; + } + ymm2 = _mm256_loadu_pd(f_temp); + ymm5 = _mm256_fmadd_pd(ymm2, ymm1, ymm5);*/ + _mm256_storeu_pd(f_temp, ymm5); + for (int i = 0; i < m_remainder; i++) + { + tC[i] = f_temp[i]; + } + } + m_remainder = 0; + } + + if (m_remainder) + { + double result; + for (; row_idx < M; row_idx += 1) + { + for (col_idx = 0; col_idx < N; col_idx += 1) + { + //pointer math to point to proper memory + tC = C + ldc * col_idx + row_idx; + tB = B + tb_inc_col * col_idx; + tA = A + row_idx; + + result = 0; + for (k = 0; k < K; ++k) + { + result += (*tA) * (*tB); + tA += lda; + tB += tb_inc_row; + } + + result *= (*alpha_cast); + (*tC) = /*(*tC) * (*beta_cast) + */result; + } + } + } + + //copy/compute sryk values back to C using SIMD + if ( bli_seq0( *beta_cast ) ) + {//just copy for beta = 0 + dim_t _i, _j, k, _l; + if(bli_obj_is_lower(c)) //c is lower + { + //first column + _j = 0; + k = M >> 2; + _i = 0; + for ( _l = 0; _l < k; _l++ ) + { + ymm0 = _mm256_loadu_pd((C + _i*rsc)); + _mm256_storeu_pd((matCbuf + _i*rs_matC), ymm0); + _i += 4; + } + while (_i < M ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + while ( _j < N ) //next column + { + //k = (_j + (4 - (_j & 3))); + _l = _j & 3; + k = (_l != 0) ? (_j + (4 - _l)) : _j; + k = (k <= M) ? k : M; + for ( _i = _j; _i < k; ++_i ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + k = (M - _i) >> 2; + _l = 0; + while ( _l < k ) + { + ymm0 = _mm256_loadu_pd((C + _i*rsc + _j*ldc)); + _mm256_storeu_pd((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + + _i += 4; + _l++; + } + while (_i < M ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + { + k = (_j + 1) >> 2; + _i = 0; + _l = 0; + while ( _l < k ) + { + ymm0 = _mm256_loadu_pd((C + _i*rsc + _j*ldc)); + _mm256_storeu_pd((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + _i += 4; + _l++; + } + while (_i <= _j ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + ++_i; + } + } + } + } + else + {//when beta is non-zero, fmadd and store the results + dim_t _i, _j, k, _l; + ymm1 = _mm256_broadcast_sd(beta_cast); + if(bli_obj_is_lower(c)) //c is lower + { + //first column + _j = 0; + k = M >> 2; + _i = 0; + for ( _l = 0; _l < k; _l++ ) + { + ymm2 = _mm256_loadu_pd((matCbuf + _i*rs_matC)); + ymm0 = _mm256_loadu_pd((C + _i*rsc)); + ymm0 = _mm256_fmadd_pd(ymm2, ymm1, ymm0); + _mm256_storeu_pd((matCbuf + _i*rs_matC), ymm0); + _i += 4; + } + while (_i < M ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + while ( _j < N ) //next column + { + //k = (_j + (4 - (_j & 3))); + _l = _j & 3; + k = (_l != 0) ? (_j + (4 - _l)) : _j; + k = (k <= M) ? k : M; + for ( _i = _j; _i < k; ++_i ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + k = (M - _i) >> 2; + _l = 0; + while ( _l < k ) + { + ymm2 = _mm256_loadu_pd((matCbuf + _i*rs_matC + _j*ldc_matC)); + ymm0 = _mm256_loadu_pd((C + _i*rsc + _j*ldc)); + ymm0 = _mm256_fmadd_pd(ymm2, ymm1, ymm0); + _mm256_storeu_pd((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + + _i += 4; + _l++; + } + while (_i < M ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + _i++; + } + _j++; + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + { + k = (_j + 1) >> 2; + _i = 0; + _l = 0; + while ( _l < k ) + { + ymm2 = _mm256_loadu_pd((matCbuf + _i*rs_matC + _j*ldc_matC)); + ymm0 = _mm256_loadu_pd((C + _i*rsc + _j*ldc)); + ymm0 = _mm256_fmadd_pd(ymm2, ymm1, ymm0); + _mm256_storeu_pd((matCbuf + _i*rs_matC + _j*ldc_matC), ymm0); + _i += 4; + _l++; + } + while (_i <= _j ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + ++_i; + } + } + } + } + + return BLIS_SUCCESS; + } + else + return BLIS_NONCONFORMAL_DIMENSIONS; + + +}; + +static err_t bli_ssyrk_small_atbn + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int M = bli_obj_length(c); // number of rows of Matrix C + int N = bli_obj_width(c); // number of columns of Matrix C + int K = bli_obj_length(b); // number of rows of Matrix B + int lda = bli_obj_col_stride(a); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + int ldb = bli_obj_col_stride(b); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + int ldc_matC = bli_obj_col_stride( c ); // column stride of matrix C + int ldc = M;//bli_obj_col_stride( c ); // column stride of static buffer for matrix C + int row_idx = 0, col_idx = 0, k; + int rs_matC = bli_obj_row_stride( c ); + int rsc = 1; + float *A = a->buffer; // pointer to matrix A elements, stored in row major format + float *B = b->buffer; // pointer to matrix B elements, stored in column major format + float *C = C_pack; // pointer to matrix C elements, stored in column major format + float *matCbuf = c->buffer; + + float *tA = A, *tB = B, *tC = C; + + __m256 ymm4, ymm5, ymm6, ymm7; + __m256 ymm8, ymm9, ymm10, ymm11; + __m256 ymm12, ymm13, ymm14, ymm15; + __m256 ymm0, ymm1, ymm2, ymm3; + + float result, scratch[8]; + float *alpha_cast, *beta_cast; // alpha, beta multiples + alpha_cast = (alpha->buffer); + beta_cast = (beta->buffer); + + // The non-copy version of the A^T SYRK gives better performance for the small M cases. + // The threshold is controlled by BLIS_ATBN_M_THRES + if (M <= BLIS_ATBN_M_THRES) + { + for (col_idx = 0; (col_idx + (NR - 1)) < N; col_idx += NR) + { + for (row_idx = 0; (row_idx + (AT_MR - 1)) < M; row_idx += AT_MR) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm5 = _mm256_setzero_ps(); + ymm6 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm8 = _mm256_setzero_ps(); + ymm9 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm11 = _mm256_setzero_ps(); + ymm12 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + ymm14 = _mm256_setzero_ps(); + ymm15 = _mm256_setzero_ps(); + + //The inner loop computes the 4x3 values of the matrix. + //The computation pattern is: + // ymm4 ymm5 ymm6 + // ymm7 ymm8 ymm9 + // ymm10 ymm11 ymm12 + // ymm13 ymm14 ymm15 + + //The Dot operation is performed in the inner loop, 8 float elements fit + //in the YMM register hence loop count incremented by 8 + for (k = 0; (k + 7) < K; k += 8) + { + ymm0 = _mm256_loadu_ps(tB + 0); + ymm1 = _mm256_loadu_ps(tB + ldb); + ymm2 = _mm256_loadu_ps(tB + 2 * ldb); + + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_ps(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_ps(ymm2, ymm3, ymm6); + + ymm3 = _mm256_loadu_ps(tA + lda); + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + ymm3 = _mm256_loadu_ps(tA + 2 * lda); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + ymm11 = _mm256_fmadd_ps(ymm1, ymm3, ymm11); + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_ps(tA + 3 * lda); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + ymm14 = _mm256_fmadd_ps(ymm1, ymm3, ymm14); + ymm15 = _mm256_fmadd_ps(ymm2, ymm3, ymm15); + + tA += 8; + tB += 8; + + } + + // if K is not a multiple of 8, padding is done before load using temproary array. + if (k < K) + { + int iter; + float data_feeder[8] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_ps(data_feeder); + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter + ldb]; + ymm1 = _mm256_loadu_ps(data_feeder); + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter + 2 * ldb]; + ymm2 = _mm256_loadu_ps(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_ps(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_ps(ymm2, ymm3, ymm6); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + ymm8 = _mm256_fmadd_ps(ymm1, ymm3, ymm8); + ymm9 = _mm256_fmadd_ps(ymm2, ymm3, ymm9); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[2 * lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + ymm11 = _mm256_fmadd_ps(ymm1, ymm3, ymm11); + ymm12 = _mm256_fmadd_ps(ymm2, ymm3, ymm12); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[3 * lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + ymm14 = _mm256_fmadd_ps(ymm1, ymm3, ymm14); + ymm15 = _mm256_fmadd_ps(ymm2, ymm3, ymm15); + + } + + //horizontal addition and storage of the data. + //Results for 4x3 blocks of C is stored here + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + _mm256_storeu_ps(scratch, ymm4); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm7 = _mm256_hadd_ps(ymm7, ymm7); + ymm7 = _mm256_hadd_ps(ymm7, ymm7); + _mm256_storeu_ps(scratch, ymm7); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm10 = _mm256_hadd_ps(ymm10, ymm10); + ymm10 = _mm256_hadd_ps(ymm10, ymm10); + _mm256_storeu_ps(scratch, ymm10); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm13 = _mm256_hadd_ps(ymm13, ymm13); + ymm13 = _mm256_hadd_ps(ymm13, ymm13); + _mm256_storeu_ps(scratch, ymm13); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + tC += ldc; + ymm5 = _mm256_hadd_ps(ymm5, ymm5); + ymm5 = _mm256_hadd_ps(ymm5, ymm5); + _mm256_storeu_ps(scratch, ymm5); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm8 = _mm256_hadd_ps(ymm8, ymm8); + ymm8 = _mm256_hadd_ps(ymm8, ymm8); + _mm256_storeu_ps(scratch, ymm8); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm11 = _mm256_hadd_ps(ymm11, ymm11); + ymm11 = _mm256_hadd_ps(ymm11, ymm11); + _mm256_storeu_ps(scratch, ymm11); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm14 = _mm256_hadd_ps(ymm14, ymm14); + ymm14 = _mm256_hadd_ps(ymm14, ymm14); + _mm256_storeu_ps(scratch, ymm14); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + tC += ldc; + ymm6 = _mm256_hadd_ps(ymm6, ymm6); + ymm6 = _mm256_hadd_ps(ymm6, ymm6); + _mm256_storeu_ps(scratch, ymm6); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm9 = _mm256_hadd_ps(ymm9, ymm9); + ymm9 = _mm256_hadd_ps(ymm9, ymm9); + _mm256_storeu_ps(scratch, ymm9); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm12 = _mm256_hadd_ps(ymm12, ymm12); + ymm12 = _mm256_hadd_ps(ymm12, ymm12); + _mm256_storeu_ps(scratch, ymm12); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm15 = _mm256_hadd_ps(ymm15, ymm15); + ymm15 = _mm256_hadd_ps(ymm15, ymm15); + _mm256_storeu_ps(scratch, ymm15); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + } + } + + int processed_col = col_idx; + int processed_row = row_idx; + + //The edge case handling where N is not a multiple of 3 + if (processed_col < N) + { + for (col_idx = processed_col; col_idx < N; col_idx += 1) + { + for (row_idx = 0; (row_idx + (AT_MR - 1)) < M; row_idx += AT_MR) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + ymm7 = _mm256_setzero_ps(); + ymm10 = _mm256_setzero_ps(); + ymm13 = _mm256_setzero_ps(); + + //The inner loop computes the 4x1 values of the matrix. + //The computation pattern is: + // ymm4 + // ymm7 + // ymm10 + // ymm13 + + for (k = 0; (k + 7) < K; k += 8) + { + ymm0 = _mm256_loadu_ps(tB + 0); + + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + ymm3 = _mm256_loadu_ps(tA + lda); + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + + ymm3 = _mm256_loadu_ps(tA + 2 * lda); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + + ymm3 = _mm256_loadu_ps(tA + 3 * lda); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + + tA += 8; + tB += 8; + } + + // if K is not a multiple of 8, padding is done before load using temproary array. + if (k < K) + { + int iter; + float data_feeder[8] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_ps(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm7 = _mm256_fmadd_ps(ymm0, ymm3, ymm7); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[2 * lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm10 = _mm256_fmadd_ps(ymm0, ymm3, ymm10); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[3 * lda + iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm13 = _mm256_fmadd_ps(ymm0, ymm3, ymm13); + + } + + //horizontal addition and storage of the data. + //Results for 4x1 blocks of C is stored here + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + _mm256_storeu_ps(scratch, ymm4); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm7 = _mm256_hadd_ps(ymm7, ymm7); + ymm7 = _mm256_hadd_ps(ymm7, ymm7); + _mm256_storeu_ps(scratch, ymm7); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm10 = _mm256_hadd_ps(ymm10, ymm10); + ymm10 = _mm256_hadd_ps(ymm10, ymm10); + _mm256_storeu_ps(scratch, ymm10); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm13 = _mm256_hadd_ps(ymm13, ymm13); + ymm13 = _mm256_hadd_ps(ymm13, ymm13); + _mm256_storeu_ps(scratch, ymm13); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + } + } + processed_row = row_idx; + } + + //The edge case handling where M is not a multiple of 4 + if (processed_row < M) + { + for (row_idx = processed_row; row_idx < M; row_idx += 1) + { + for (col_idx = 0; col_idx < N; col_idx += 1) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_ps(); + + for (k = 0; (k + 7) < K; k += 8) + { + ymm0 = _mm256_loadu_ps(tB + 0); + ymm3 = _mm256_loadu_ps(tA); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + tA += 8; + tB += 8; + } + + // if K is not a multiple of 8, padding is done before load using temproary array. + if (k < K) + { + int iter; + float data_feeder[8] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_ps(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_ps(data_feeder); + ymm4 = _mm256_fmadd_ps(ymm0, ymm3, ymm4); + + } + + //horizontal addition and storage of the data. + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + ymm4 = _mm256_hadd_ps(ymm4, ymm4); + _mm256_storeu_ps(scratch, ymm4); + result = scratch[0] + scratch[4]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + } + } + } + + //copy/compute sryk values back to C + if ( bli_seq0( *beta_cast ) ) //when beta is 0, just copy result to C + { + dim_t _i, _j; + if(bli_obj_is_lower(c)) //c is lower + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i <= 0 ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i >= 0 ) + { + bli_sscopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + } + else //when beta is non-zero, multiply and store result to C + { + dim_t _i, _j; + if(bli_obj_is_lower(c)) //c is lower + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i <= 0 ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i >= 0 ) + { + bli_sssxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + } + + return BLIS_SUCCESS; + } + else + return BLIS_NONCONFORMAL_DIMENSIONS; +} + +static err_t bli_dsyrk_small_atbn + ( + obj_t* alpha, + obj_t* a, + obj_t* b, + obj_t* beta, + obj_t* c, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int M = bli_obj_length( c ); // number of rows of Matrix C + int N = bli_obj_width( c ); // number of columns of Matrix C + int K = bli_obj_length( b ); // number of rows of Matrix B + int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + int ldc_matC = bli_obj_col_stride( c ); // column stride of matrix C + int ldc = M;//bli_obj_col_stride( c ); // column stride of static buffer for matrix C + int row_idx = 0, col_idx = 0, k; + int rs_matC = bli_obj_row_stride( c ); + int rsc = 1; + double *A = a->buffer; // pointer to matrix A elements, stored in row major format + double *B = b->buffer; // pointer to matrix B elements, stored in column major format + double *C = D_C_pack; // pointer to matrix C elements, stored in column major format + double *matCbuf = c->buffer; + + double *tA = A, *tB = B, *tC = C; + + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm0, ymm1, ymm2, ymm3; + + double result, scratch[8]; + double *alpha_cast, *beta_cast; // alpha, beta multiples + alpha_cast = (alpha->buffer); + beta_cast = (beta->buffer); + + // The non-copy version of the A^T SYRK gives better performance for the small M cases. + // The threshold is controlled by BLIS_ATBN_M_THRES + if (M <= BLIS_ATBN_M_THRES) + { + for (col_idx = 0; (col_idx + (NR - 1)) < N; col_idx += NR) + { + for (row_idx = 0; (row_idx + (AT_MR - 1)) < M; row_idx += AT_MR) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + //The inner loop computes the 4x3 values of the matrix. + //The computation pattern is: + // ymm4 ymm5 ymm6 + // ymm7 ymm8 ymm9 + // ymm10 ymm11 ymm12 + // ymm13 ymm14 ymm15 + + //The Dot operation is performed in the inner loop, 4 double elements fit + //in the YMM register hence loop count incremented by 4 + for (k = 0; (k + 3) < K; k += 4) + { + ymm0 = _mm256_loadu_pd(tB + 0); + ymm1 = _mm256_loadu_pd(tB + ldb); + ymm2 = _mm256_loadu_pd(tB + 2 * ldb); + + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_pd(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_pd(ymm2, ymm3, ymm6); + + ymm3 = _mm256_loadu_pd(tA + lda); + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + ymm3 = _mm256_loadu_pd(tA + 2 * lda); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + ymm11 = _mm256_fmadd_pd(ymm1, ymm3, ymm11); + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + ymm3 = _mm256_loadu_pd(tA + 3 * lda); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + ymm14 = _mm256_fmadd_pd(ymm1, ymm3, ymm14); + ymm15 = _mm256_fmadd_pd(ymm2, ymm3, ymm15); + + tA += 4; + tB += 4; + + } + + // if K is not a multiple of 4, padding is done before load using temproary array. + if (k < K) + { + int iter; + double data_feeder[4] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_pd(data_feeder); + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter + ldb]; + ymm1 = _mm256_loadu_pd(data_feeder); + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter + 2 * ldb]; + ymm2 = _mm256_loadu_pd(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + ymm5 = _mm256_fmadd_pd(ymm1, ymm3, ymm5); + ymm6 = _mm256_fmadd_pd(ymm2, ymm3, ymm6); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + ymm8 = _mm256_fmadd_pd(ymm1, ymm3, ymm8); + ymm9 = _mm256_fmadd_pd(ymm2, ymm3, ymm9); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[2 * lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + ymm11 = _mm256_fmadd_pd(ymm1, ymm3, ymm11); + ymm12 = _mm256_fmadd_pd(ymm2, ymm3, ymm12); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[3 * lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + ymm14 = _mm256_fmadd_pd(ymm1, ymm3, ymm14); + ymm15 = _mm256_fmadd_pd(ymm2, ymm3, ymm15); + + } + + //horizontal addition and storage of the data. + //Results for 4x3 blocks of C is stored here + ymm4 = _mm256_hadd_pd(ymm4, ymm4); + _mm256_storeu_pd(scratch, ymm4); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm7 = _mm256_hadd_pd(ymm7, ymm7); + _mm256_storeu_pd(scratch, ymm7); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm10 = _mm256_hadd_pd(ymm10, ymm10); + _mm256_storeu_pd(scratch, ymm10); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm13 = _mm256_hadd_pd(ymm13, ymm13); + _mm256_storeu_pd(scratch, ymm13); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + + tC += ldc; + ymm5 = _mm256_hadd_pd(ymm5, ymm5); + _mm256_storeu_pd(scratch, ymm5); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm8 = _mm256_hadd_pd(ymm8, ymm8); + _mm256_storeu_pd(scratch, ymm8); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm11 = _mm256_hadd_pd(ymm11, ymm11); + _mm256_storeu_pd(scratch, ymm11); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm14 = _mm256_hadd_pd(ymm14, ymm14); + _mm256_storeu_pd(scratch, ymm14); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + + tC += ldc; + ymm6 = _mm256_hadd_pd(ymm6, ymm6); + _mm256_storeu_pd(scratch, ymm6); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm9 = _mm256_hadd_pd(ymm9, ymm9); + _mm256_storeu_pd(scratch, ymm9); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm12 = _mm256_hadd_pd(ymm12, ymm12); + _mm256_storeu_pd(scratch, ymm12); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm15 = _mm256_hadd_pd(ymm15, ymm15); + _mm256_storeu_pd(scratch, ymm15); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + } + } + + int processed_col = col_idx; + int processed_row = row_idx; + + //The edge case handling where N is not a multiple of 3 + if (processed_col < N) + { + for (col_idx = processed_col; col_idx < N; col_idx += 1) + { + for (row_idx = 0; (row_idx + (AT_MR - 1)) < M; row_idx += AT_MR) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + + //The inner loop computes the 4x1 values of the matrix. + //The computation pattern is: + // ymm4 + // ymm7 + // ymm10 + // ymm13 + + for (k = 0; (k + 3) < K; k += 4) + { + ymm0 = _mm256_loadu_pd(tB + 0); + + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + ymm3 = _mm256_loadu_pd(tA + lda); + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + + ymm3 = _mm256_loadu_pd(tA + 2 * lda); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + + ymm3 = _mm256_loadu_pd(tA + 3 * lda); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + + tA += 4; + tB += 4; + } + // if K is not a multiple of 4, padding is done before load using temproary array. + if (k < K) + { + int iter; + double data_feeder[4] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_pd(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm7 = _mm256_fmadd_pd(ymm0, ymm3, ymm7); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[2 * lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm10 = _mm256_fmadd_pd(ymm0, ymm3, ymm10); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[3 * lda + iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm13 = _mm256_fmadd_pd(ymm0, ymm3, ymm13); + + } + + //horizontal addition and storage of the data. + //Results for 4x1 blocks of C is stored here + ymm4 = _mm256_hadd_pd(ymm4, ymm4); + _mm256_storeu_pd(scratch, ymm4); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + ymm7 = _mm256_hadd_pd(ymm7, ymm7); + _mm256_storeu_pd(scratch, ymm7); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[1] = result/* + tC[1] * (*beta_cast)*/; + + ymm10 = _mm256_hadd_pd(ymm10, ymm10); + _mm256_storeu_pd(scratch, ymm10); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[2] = result/* + tC[2] * (*beta_cast)*/; + + ymm13 = _mm256_hadd_pd(ymm13, ymm13); + _mm256_storeu_pd(scratch, ymm13); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[3] = result/* + tC[3] * (*beta_cast)*/; + + } + } + processed_row = row_idx; + } + + // The edge case handling where M is not a multiple of 4 + if (processed_row < M) + { + for (row_idx = processed_row; row_idx < M; row_idx += 1) + { + for (col_idx = 0; col_idx < N; col_idx += 1) + { + tA = A + row_idx * lda; + tB = B + col_idx * ldb; + tC = C + col_idx * ldc + row_idx; + // clear scratch registers. + ymm4 = _mm256_setzero_pd(); + + for (k = 0; (k + 3) < K; k += 4) + { + ymm0 = _mm256_loadu_pd(tB + 0); + ymm3 = _mm256_loadu_pd(tA); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + tA += 4; + tB += 4; + } + + // if K is not a multiple of 4, padding is done before load using temproary array. + if (k < K) + { + int iter; + double data_feeder[4] = { 0.0 }; + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tB[iter]; + ymm0 = _mm256_loadu_pd(data_feeder); + + for (iter = 0; iter < (K - k); iter++) data_feeder[iter] = tA[iter]; + ymm3 = _mm256_loadu_pd(data_feeder); + ymm4 = _mm256_fmadd_pd(ymm0, ymm3, ymm4); + + } + + //horizontal addition and storage of the data. + ymm4 = _mm256_hadd_pd(ymm4, ymm4); + _mm256_storeu_pd(scratch, ymm4); + result = scratch[0] + scratch[2]; + result *= (*alpha_cast); + tC[0] = result/* + tC[0] * (*beta_cast)*/; + + } + } + } + + //copy/compute sryk values back to C + if ( bli_seq0( *beta_cast ) ) //when beta is 0, just copy result to C + { + dim_t _i, _j; + if(bli_obj_is_lower(c)) //c is lower + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i <= 0 ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i >= 0 ) + { + bli_ddcopys( *(C + _i*rsc + _j*ldc), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + } + else //when beta is non-zero, multiply and store result to C + { + dim_t _i, _j; + if(bli_obj_is_lower(c)) //c is lower + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i <= 0 ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + else //c is upper + { + for ( _j = 0; _j < N; ++_j ) + for ( _i = 0; _i < M; ++_i ) + if ( (doff_t)_j - (doff_t)_i >= 0 ) + { + bli_dddxpbys( *(C + _i*rsc + _j*ldc), + *(beta_cast), + *(matCbuf + _i*rs_matC + _j*ldc_matC) ); + } + } + } + + return BLIS_SUCCESS; + } + else + return BLIS_NONCONFORMAL_DIMENSIONS; +} + +#endif + diff --git a/kernels/zen/3/bli_trsm_small.c b/kernels/zen/3/bli_trsm_small.c new file mode 100644 index 000000000..ca8e5b142 --- /dev/null +++ b/kernels/zen/3/bli_trsm_small.c @@ -0,0 +1,15021 @@ +/* + +BLIS +An object-based framework for developing high-performance BLAS-like +libraries. + +Copyright (C) 2018, Advanced Micro Devices, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of The University of Texas at Austin nor the names +of its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" +#ifdef BLIS_ENABLE_SMALL_MATRIX_TRSM +#include "immintrin.h" + +#define GEMM_BLK_V1 8 //Block size to perform gemm and apply trsm +#define GEMM_ACCUM_A 1 //Peform B1=B1-(B0*A0) operation instead of B1'=(B0*A0) and then B1=B1-B1' +#define OPT_CACHE_BLOCKING_L1 1 //Perform trsm block-wise in blocks of GEMM_BLK_V1 instead of all columns of B together. +#define REARRANGE_SHFL 0 //Rearrange operations using blend or shuffle +#define BLI_AlXB_M_SP 16 +#define BLI_AlXB_M_DP 16 +#define BLI_XAltB_N_SP 128 +#define BLI_XAltB_N_DP 64 +#define BLI_AutXB_M_SP 64 +#define BLI_AutXB_N_SP 128 + +static void (*fp_blis_strsm_microkernel)( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); +static void blis_strsm_microkernel( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); +static void blis_strsm_microkernel_alpha( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal + ); +static void blis_strsm_microkernel_unitDiag( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); +static void blis_strsm_microkernel_alpha_unitDiag( float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal + ); +static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal); +static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alphaVal); + +static void (*fp_blis_dtrsm_microkernel)( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); + +static void blis_dtrsm_microkernel( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); + +static void blis_dtrsm_microkernel_alpha( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ); + +static void blis_dtrsm_microkernel_unitDiag( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ); + +static void blis_dtrsm_microkernel_alpha_unitDiag( double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ); + +static void dtrsm_XAtB_block_allSmallSizedMatrices(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal); +static void dtrsm_XAtB_block_allSmallSizedMatrices_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal); +static void trsm_AutXB_block_allSmallSizedMatrices(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void trsm_AutXB_block_allSmallSizedMatrices_alpha(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alpha); +static void trsm_AutXB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b); +static void trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, + float *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + float alpha); + +//AX = B; A is lower triangular; No transpose; single precision +static err_t bli_strsm_small_AlXB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); +//A.'X = B; A is upper triangular; A has to be transposed; single precision +static err_t bli_strsm_small_AutXB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + +//XA.' = B; A is lower triangular; A has to be transposed; single precision +static err_t bli_strsm_small_XAltB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); +//AX = B; A is lower triangular; No transpose; double precision +static err_t bli_dtrsm_small_AlXB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + + +//A.'X = B; A is upper triangular; A has to be transposed; double precision +static err_t bli_dtrsm_small_AutXB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + + +//XA.' = B; A is lower triangular; A has to be transposed; double precision +static err_t bli_dtrsm_small_XAltB + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ); + void trsm_block_c(float *ptr_l, float *ptr_b, int blk_height, int blk_width, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b); +/* +* The bli_trsm_small implements unpacked version of TRSM +* Currently only column-major is supported, A & B are column-major +* Input: A: MxM (triangular matrix) +* B: MxN matrix +* Output: X: MxN matrix such that AX = alpha*B or XA = alpha*B or A'X = alpha*B or XA' = alpha*B +* Here the output X is stored in B +* The custom-kernel will be called only when M*(M+N)* sizeof(Matrix Elements) < L3 cache +*/ +err_t bli_trsm_small + ( + side_t side, + obj_t* alpha, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ +#ifdef BLIS_ENABLE_MULTITHREADING + return BLIS_NOT_YET_IMPLEMENTED; +#endif + + // If alpha is zero, B matrix will become zero after scaling & hence solution is also zero matrix + if (bli_obj_equals(alpha, &BLIS_ZERO)) + { + return BLIS_NOT_YET_IMPLEMENTED; // scale B by alpha + } + // We have to call matrix scaling if alpha != 1.0 + + // if row major format return. Check this again. + if ((bli_obj_row_stride(a) != 1) || + (bli_obj_row_stride(b) != 1)) + { + return BLIS_INVALID_ROW_STRIDE; + } + + num_t dt = ((*b).info & (0x7 << 0)); + + // only float and double datatypes are supported as of now. + if (dt != BLIS_DOUBLE && dt != BLIS_FLOAT) + { + return BLIS_EXPECTED_REAL_DATATYPE; + } + + // A is expected to be triangular in trsm + if (!bli_obj_is_upper_or_lower (a)) + { + return BLIS_EXPECTED_TRIANGULAR_OBJECT; + } + + // can use other control structs - even can use array of function pointers, + // indexed by a number with bits formed by f('side', 'uplo', 'transa', dt). + // In the below implementation, based on the number of finally implemented + // cases, can move the checks with more cases higher up. + if (side == BLIS_LEFT) + { + if (bli_obj_has_trans(a)) + { + if (dt == BLIS_DOUBLE) + { + if (bli_obj_is_upper(a)) + { + //A.'X = B; A is upper triangular; A has to be transposed; double precision +#if 0 // planning to implement this in this iteration + return bli_dtrsm_small_AutXB(side, alpha, a, b, cntx, cntl); +#else + return BLIS_NOT_YET_IMPLEMENTED; +#endif + } + else + { + return BLIS_NOT_YET_IMPLEMENTED; + } + } + else if (dt == BLIS_FLOAT) + { + if (bli_obj_is_upper(a)) + { + //A.'X = B; A is upper triangular; A has to be transposed; single precision + return bli_strsm_small_AutXB(side, alpha, a, b, cntx, cntl); + } + else + { + return BLIS_NOT_YET_IMPLEMENTED; + } + } + } + else + { + if (dt == BLIS_DOUBLE) + { + if (bli_obj_is_upper(a)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //AX = B; A is lower triangular; No transpose; double precision + return bli_dtrsm_small_AlXB(side, alpha, a, b, cntx, cntl); + } + } + else if (dt == BLIS_FLOAT) + { + if (bli_obj_is_upper(a)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //AX = B; A is lower triangular; No transpose; single precision + return bli_strsm_small_AlXB(side, alpha, a, b, cntx, cntl); + } + } + } + } + else + { + if (bli_obj_has_trans(a)) + { + if (dt == BLIS_DOUBLE) + { + if (bli_obj_is_upper(a)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //XA.' = B; A is lower triangular; A has to be transposed; double precision + return bli_dtrsm_small_XAltB(side, alpha, a, b, cntx, cntl); + } + } + else if (dt == BLIS_FLOAT) + { + if (bli_obj_is_upper(a)) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + else + { + //XA.' = B; A is lower triangular; A has to be transposed; single precision + return bli_strsm_small_XAltB(side, alpha, a, b, cntx, cntl); + } + } + } + else + { + return BLIS_NOT_YET_IMPLEMENTED; + } + } + + return BLIS_NOT_YET_IMPLEMENTED; +}; + + +static void trsm_small_AlXB ( + float *A, + float *B, + int M, + int N, + int lda, + int ldb + ) +{ + int i; + int j; + int k; + + // Need to incorporate alpha + + for (k = 0; k < M; k++) + { + float lkk_inv = 1.0/A[k+k*lda]; + + for (j = 0; j < N; j++) + { + B[k + j*ldb] *= lkk_inv; + + for (i = k+1; i < M; i++) + { + B[i + j*ldb] -= A[i + k*lda] * B[k + j*ldb]; + } + } + }// k -loop + +}// end of function + + +// Test code: +void gemm_small( float *ptr_l, + float *ptr_b, + int blk_m, + int blk_n, + float *ptr_gemmOut, + int cs_l, + int cs_b, + int rs_l, + int rs_b, + float alpha, + float beta) +{ + int i, j, k; + + for (i = 0; i < blk_m; i++) + { + for (j = 0; j < blk_n; j++) + { + float t = 0.0; + for (k = 0; k < blk_m; k++) + { + t += (ptr_l[i*rs_l + k* cs_l] * ptr_b[k*rs_b + j*cs_b]); + } + ptr_gemmOut[i*rs_b + j*cs_b] = beta * ptr_gemmOut[i*rs_b + j*cs_b] + alpha * t; + } + } +} + +/* + * AX = Alpha*B, Double precision, A:lower triangular + * THIS KERNEL SUPPORTS MATRIX SIZE OF THE FORM BLI_AlXB_M_DPX4*i, WHERE i IS AN INTEGER + */ + +static err_t bli_dtrsm_small_AlXB ( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + obj_t alpha, beta; // gemm parameters + obj_t Ga, Gb, Gc; // for GEMM + int m = bli_obj_length(b); // number of rows of matrix B + int n = bli_obj_width(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int j; + int blk_size = 4; + int isUnitDiag = bli_obj_has_unit_diag(a); + + double alphaVal; + double *L = a->buffer; + double *B = b->buffer; + + if (m != BLI_AlXB_M_DP || (n&3) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((double *)AlphaObj->buffer); + + /* Small _GEMM preparation code */ + bli_obj_create( BLIS_DOUBLE, 1, 1, 0, 0, &alpha ); + bli_obj_create( BLIS_DOUBLE, 1, 1, 0, 0, &beta ); + + /* B = B - A*B */ + bli_setsc( -(1.0), 0.0, &alpha ); + bli_setsc( (1.0), 0.0, &beta ); + + bli_obj_create_with_attached_buffer( BLIS_DOUBLE, blk_size, blk_size, a->buffer, rsa, lda, &Ga); + bli_obj_create_with_attached_buffer( BLIS_DOUBLE, blk_size, n, b->buffer, rsb, ldb, &Gb); + bli_obj_create_with_attached_buffer( BLIS_DOUBLE, blk_size, n, b->buffer, rsb, ldb, &Gc); + + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Ga ); + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gb ); + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gc ); + + //first block of trsm + Gb.buffer = (void*)(B + i); + + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + blis_dtrsm_microkernel_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_dtrsm_microkernel = blis_dtrsm_microkernel; + } + else + { + blis_dtrsm_microkernel_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_dtrsm_microkernel = blis_dtrsm_microkernel_unitDiag; + } + bli_setsc( alphaVal, 0.0, &beta ); + } + else + { + if (isUnitDiag == 0) + { + blis_dtrsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_dtrsm_microkernel = blis_dtrsm_microkernel; + } + else + { + blis_dtrsm_microkernel_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_dtrsm_microkernel = blis_dtrsm_microkernel_unitDiag; + } + } + + +//gemm update + for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + { + Ga.buffer = (void*)(L + j + i*lda); + Gc.buffer = (void*)(B + j); + bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + } + bli_setsc( (1.0), 0.0, &beta ); + + //trsm of remaining blocks + for (i = blk_size; i < m; i += blk_size) + { + Gb.buffer = (void*)(B + i); + + fp_blis_dtrsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + + + for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + { + Ga.buffer = (void*)(L + j + i*lda); + Gc.buffer = (void*)(B + j); + + bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + } + + } // End of for loop - i + + return BLIS_SUCCESS; + +} + + +/* + * AX = Alpha*B, Single precision, A: lower triangular + * This kernel implementation supports matrices A and B such that m is equal to BLI_AlXB_M_SP and n is mutiple of 8 + */ +static err_t bli_strsm_small_AlXB ( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + obj_t alpha, beta; // gemm parameters + obj_t Ga, Gb, Gc; // for GEMM + int m = bli_obj_length(b); // number of rows of matrix B + int n = bli_obj_width(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int j; + int blk_size = 8; + int isUnitDiag = bli_obj_has_unit_diag(a); + + float alphaVal; + float *L = a->buffer; + float *B = b->buffer; + + if (m != BLI_AlXB_M_SP || (n&7) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); + + /* Small _GEMM preparation code */ + bli_obj_create( BLIS_FLOAT, 1, 1, 0, 0, &alpha ); + bli_obj_create( BLIS_FLOAT, 1, 1, 0, 0, &beta ); + + /* B = B - A*B */ + bli_setsc( -(1.0), 0.0, &alpha ); + bli_setsc( (1.0), 0.0, &beta ); + + + bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, blk_size, a->buffer, rsa, lda, &Ga); + bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, n, b->buffer, rsb, ldb, &Gb); + bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, n, b->buffer, rsb, ldb, &Gc); + + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Ga ); + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gb ); + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gc ); + + //first block of trsm + Gb.buffer = (void*)(B + i); + + //trsm of first 8xn block + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + blis_strsm_microkernel_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_strsm_microkernel = blis_strsm_microkernel; + } + else + { + blis_strsm_microkernel_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; + } + bli_setsc( alphaVal, 0.0, &beta ); + } + else + { + if (isUnitDiag == 0) + { + blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_strsm_microkernel = blis_strsm_microkernel; + } + else + { + blis_strsm_microkernel_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; + } + } + + //gemm update + for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + { + Ga.buffer = (void*)(L + j + i*lda); + Gc.buffer = (void*)(B + j); + + bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + } + + //trsm of remaining blocks + for (i = blk_size; i < m; i += blk_size) + { + Gb.buffer = (void*)(B + i); + + fp_blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + + for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + { + Ga.buffer = (void*)(L + j + i*lda); + Gc.buffer = (void*)(B + j); + + bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + } + + } // End of for loop - i + + return BLIS_SUCCESS; +} + +void trsm_block_c(float *ptr_l, float *ptr_b, int blk_height, int blk_width, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + int i, j, k, l; + float inv_l; + + inv_l = 1.0 / *ptr_l; + + for (j = 0; j < numCols_b; j += blk_width) + { + for (l = j; l < (j+blk_width); l++) + { + ptr_b[l*cs_b] = ptr_b[l*cs_b] * inv_l; + } + + for (i = 1; i < blk_height; i++) + { + for (l = j; l < (j+blk_width); l++) + { + for (k = 0; k < i; k++) + { + ptr_b[i*rs_b + l*cs_b] -= (ptr_b[k*rs_b + l*cs_b] * ptr_l[i*rs_l + k*cs_l]); + } + ptr_b[i*rs_b + l*cs_b] = ptr_b[i*rs_b + l*cs_b] / ptr_l[i*rs_l + i*cs_l]; + } + } + } +} + + +/* + * XA' = Alpha*B, Double precision, A:lower triangular + * This kernel implementation supports matrices A and B such that + * m and n are multiples of 4 and n less than or equal to BLI_XAltB_N_DP + */ + +static err_t bli_dtrsm_small_XAltB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + int m = bli_obj_length(a); // number of rows of matrix B + int n = bli_obj_length(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int isUnitDiag = bli_obj_has_unit_diag(a); + + double alphaVal; + double *L = a->buffer; + double *B = b->buffer; + + if ((m&3) != 0 || (n&3) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( n > BLI_XAltB_N_DP || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + alphaVal = *((double *)AlphaObj->buffer); + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + dtrsm_XAtB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + else + { + dtrsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + } + else + { + if (isUnitDiag == 0) + { + dtrsm_XAtB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + else + { + dtrsm_XAtB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + } + return BLIS_SUCCESS; + +} + + +/* + * XA' = Alpha*B, Single precision, A: lower triangular + * This kernel implementation supports matrices A and B such that + * m and n are multiples of 8 and n is less than or equal to BLI_XAltB_N_SP + */ +static err_t bli_strsm_small_XAltB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int m = bli_obj_length(a); // number of rows of matrix B + int n = bli_obj_length(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int isUnitDiag = bli_obj_has_unit_diag(a); + + float alphaVal; + float *L = a->buffer; + float *B = b->buffer; + + if ((m&7) != 0 || (n&7) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( n > BLI_XAltB_N_SP || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); + + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + trsm_XAtB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + else + { + trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + } + else + { + if (isUnitDiag == 0) + { + trsm_XAtB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + else + { + trsm_XAtB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + } + return BLIS_SUCCESS; +} + +/* + * A'X = Alpha*B, Single precision, A: upper triangular + * This kernel implementation supports matrices A and B such that + * m and n are multiples of 8, m is less than or equal to BLI_AutXB_M_SP and n is less than or equal to BLI_AutXB_N_SP + */ +static err_t bli_strsm_small_AutXB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int m = bli_obj_width(a); // number of rows of matrix A (since At, so width is taken) + int n = bli_obj_width(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int isUnitDiag = bli_obj_has_unit_diag(a); + + float alphaVal; + float *L = a->buffer; + float *B = b->buffer; + + if ((m&7) != 0 || (n&7) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( m > BLI_AutXB_M_SP || n > BLI_AutXB_N_SP || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); + + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + trsm_AutXB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + else + { + trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + } + else + { + if (isUnitDiag == 0) + { + trsm_AutXB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + else + { + trsm_AutXB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + } + return BLIS_SUCCESS; +} +/* +* AX=B A=LOWER TRIANGULAR, NO TRANSPOSE, NON-UNITDIAGONAL +* ALPHA != 1; +*/ +static void blis_dtrsm_microkernel_alpha(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ) +{ + double ones = 1.0; + int j; + int cs_b_offset[2]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags; + __m256d alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + + reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); + alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); + + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + numCols_b -= 4; // blk_width = 4 + + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); + reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); + + for(j = 0;j < numCols_b; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + } + //Last block trsm processing + + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + +} +/* +*AX=B A=LOWER TRIANGULAR, NO TRANSPOSE, UNITDIAGONAL +*ALPHA != 1; +*/ +static void blis_dtrsm_microkernel_alpha_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ) +{ + + int j; + int cs_b_offset[2]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + __m256d alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + + alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + numCols_b -= 4; // blk_width = 4 + + for(j = 0;j < numCols_b; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + } + //Last block trsm processing + + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + +} +/* +*AX = B A= LOWERTRIANGULAR, NO TRANSPOSE, NON-UNITDIAGONAL +*ALPHA = 1 +*/ +static void blis_dtrsm_microkernel(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ) +{ + double ones = 1.0; + int j; + int cs_b_offset[2]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + + reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + numCols_b -= 4; // blk_width = 4 + + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); + reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); + + for(j = 0;j < numCols_b; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + } + //Last block trsm processing + + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); +} +/* +*AX = B A=LOWER TRIANGULAR, NO TRANSPOSE, UNITDIAGONAL +*ALPHA = 1 +*/ +static void blis_dtrsm_microkernel_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ) +{ + //double ones = 1.0; + int j; + int cs_b_offset[2]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + numCols_b -= 4; // blk_width = 4 + + + for(j = 0;j < numCols_b; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + } + //Last block trsm processing + + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); +} +///////////////////////////// AX=B /////////////////////////////// +static void blis_strsm_microkernel_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) +{ + float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags; + __m256 alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols +} + +static void blis_strsm_microkernel_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) +{ + //float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags; + __m256 alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //8th col + //ptr_l += cs_l; + //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols +} + +static void blis_strsm_microkernel_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //8th col + //ptr_l += cs_l; + //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols +} + +static void blis_strsm_microkernel(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols +} + +///////////////////////////////////// XA'=B functions //////////////////////////////// +static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alpha + ) + +{ + + double ones = 1.0; + int i,i1,i2,i3,i4,j,k,l; + int cs_b_offset[3]; + int cs_l_offset[3]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[16][4]; + __m256d mat_a_cols_rearr[4]; + __m256d mat_a_blk_elems[16]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags[2]; + __m256d alphaReg; + reciprocal_diags[0] = _mm256_broadcast_sd((double const *)(&ones)); + alphaReg = _mm256_broadcast_sd((double const *)&alpha); + + // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + + //read diag elems of L 4x4 block + mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[1]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 4 diags together + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); + + //Broadcast A32 to register + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + /***************** first set of 4 cols of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 4) + { + /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A + //read 4x4 block of B into registers + + mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + mat_b_rearr[0][0] = _mm256_mul_pd(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_pd(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_pd(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_pd(mat_b_rearr[3][0], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_pd(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_pd(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_pd(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_pd(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + + i += 4; + ptr_b_dup += 4; + + } + + /***************** first set of 4 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width} + + for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row + { + ptr_l += 4; + ptr_b_dup += cs_b_offset[2]; + i1 += cs_b_offset[2]; + //printf("i1 = i3 = %g\n",*(ptr_l+i1)); + //Read next 4x4 block of A to get diag elements + i3 += cs_l_offset[2]; + mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[1]); + + //pack 4 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 + + //reciprocal of diagnal elements of A :- 0,1,2,3 + reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 4) + { + i = i1 + k; + mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_pd(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_pd(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_pd(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_pd(mat_b_rearr[i2][3], alphaReg); + i2++; + } + + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 4) // move across m + { + + //Broadcast A4,0 to A7,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + + //Broadcast A41 to A71 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); + + //Broadcast A4,2 to A7,2 to registers + mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); + + //Broadcast A4,3 to A7,3 to registers + mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); + + i += cs_l_offset[2]; + + for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); + + + i4 = k >> 2; + + //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + + + //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + //end loop of cols + + } + i2 += cs_b_offset[2]; + + } + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + k = 0; + for (i = 0; i < numCols_b; i+=4) + { + + + + /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_pd(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_pd(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_pd(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_pd(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //Store the computed B columns + + _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + + k++; + } + } + +} + +static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alpha + ) + +{ + + int i,i1,i2,i3,i4,j,k,l; + int cs_b_offset[3]; + int cs_l_offset[3]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[16][4]; + __m256d mat_a_blk_elems[16]; + __m256d alphaReg; + alphaReg = _mm256_broadcast_sd((double const *)&alpha); + + // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); + + //Broadcast A32 to register + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); + + /***************** first set of 4 cols of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 4) + { + /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A + //read 4x4 block of B into registers + + mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + mat_b_rearr[0][0] = _mm256_mul_pd(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_pd(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_pd(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_pd(mat_b_rearr[3][0], alphaReg); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[0][0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[0][0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[0][0], mat_b_rearr[3][0]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[1][0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[1][0], mat_b_rearr[3][0]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_rearr[2][0], mat_b_rearr[3][0]);//d = c - (a*b) + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_b_rearr[0][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_rearr[1][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3][0]); + + i += 4; + ptr_b_dup += 4; + + } + + /***************** first set of 4 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width} + + for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row + { + ptr_l += 4; + ptr_b_dup += cs_b_offset[2]; + i1 += cs_b_offset[2]; + i3 += cs_l_offset[2]; + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 4) + { + i = i1 + k; + mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_pd(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_pd(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_pd(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_pd(mat_b_rearr[i2][3], alphaReg); + i2++; + } + + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 4) // move across m + { + + //Broadcast A4,0 to A7,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + + //Broadcast A41 to A71 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); + + //Broadcast A4,2 to A7,2 to registers + mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); + + //Broadcast A4,3 to A7,3 to registers + mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); + + i += cs_l_offset[2]; + + for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); + + + i4 = k >> 2; + + //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + + + //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + //end loop of cols + + } + i2 += cs_b_offset[2]; + + } + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + k = 0; + for (i = 0; i < numCols_b; i+=4) + { + + + + /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + + //Store the computed B columns + + _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + + k++; + } + + } + + +} + +static void dtrsm_XAtB_block_allSmallSizedMatrices_unitDiag(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ) + +{ + + int i,i1,i2,i3,i4,j,k,l; + int cs_b_offset[3]; + int cs_l_offset[3]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[16][4]; + __m256d mat_a_blk_elems[16]; + + // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); + + //Broadcast A32 to register + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); + + /***************** first set of 4 cols of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 4) + { + /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A + //read 4x4 block of B into registers + + mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[0][0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[0][0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[0][0], mat_b_rearr[3][0]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[1][0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[1][0], mat_b_rearr[3][0]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_rearr[2][0], mat_b_rearr[3][0]);//d = c - (a*b) + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_b_rearr[0][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_rearr[1][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3][0]); + + i += 4; + ptr_b_dup += 4; + + } + + /***************** first set of 4 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width} + + for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row + { + ptr_l += 4; + ptr_b_dup += cs_b_offset[2]; + i1 += cs_b_offset[2]; + i3 += cs_l_offset[2]; + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 4) + { + i = i1 + k; + mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + i2++; + } + + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 4) // move across m + { + + //Broadcast A4,0 to A7,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + + //Broadcast A41 to A71 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); + + //Broadcast A4,2 to A7,2 to registers + mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); + + //Broadcast A4,3 to A7,3 to registers + mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); + + i += cs_l_offset[2]; + + for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); + + + i4 = k >> 2; + + //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + + + //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + //end loop of cols + + } + i2 += cs_b_offset[2]; + + } + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + k = 0; + for (i = 0; i < numCols_b; i+=4) + { + + + + /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + + //Store the computed B columns + + _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + + k++; + } + + } + +} +static void dtrsm_XAtB_block_allSmallSizedMatrices(double *ptr_l, + double *ptr_b, + int numRows_lb, + int numCols_b, + int rs_l, + int rs_b, + int cs_l, + int cs_b + ) + +{ + + double ones = 1.0; + int i,i1,i2,i3,i4,j,k,l; + int cs_b_offset[3]; + int cs_l_offset[3]; + double *ptr_b_dup; + + __m256d mat_b_col[4]; + __m256d mat_b_rearr[16][4]; + __m256d mat_a_cols_rearr[4]; + __m256d mat_a_blk_elems[16]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_sd((double const *)(&ones)); + + // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + + //read diag elems of L 4x4 block + mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[1]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 4 diags together + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); + + //Broadcast A32 to register + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + /***************** first set of 4 cols of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 4) + { + /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A + //read 4x4 block of B into registers + + mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_pd(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_pd(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_pd(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_pd(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //Store the computed B columns + _mm256_storeu_pd((double *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + + i += 4; + ptr_b_dup += 4; + + } + + /***************** first set of 4 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width} + + for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row + { + ptr_l += 4; + ptr_b_dup += cs_b_offset[2]; + i1 += cs_b_offset[2]; + //printf("i1 = i3 = %g\n",*(ptr_l+i1)); + //Read next 4x4 block of A to get diag elements + i3 += cs_l_offset[2]; + mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[1]); + + //pack 4 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 + + //reciprocal of diagnal elements of A :- 0,1,2,3 + reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 4) + { + i = i1 + k; + mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + + i2++; + } + + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 4) // move across m + { + + //Broadcast A4,0 to A7,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + + //Broadcast A41 to A71 to registers + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); + + //Broadcast A4,2 to A7,2 to registers + mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); + + //Broadcast A4,3 to A7,3 to registers + mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); + + i += cs_l_offset[2]; + + for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); + + + i4 = k >> 2; + + //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + + + //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + //end loop of cols + + } + i2 += cs_b_offset[2]; + + } + + //Broadcast A10 to A30 to registers + mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //Broadcast A21 to A31 to registers + mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); + mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); + i += cs_l; + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + k = 0; + for (i = 0; i < numCols_b; i+=4) + { + + + + /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_pd(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_pd(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_pd(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_pd(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //Store the computed B columns + + _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + + k++; + } + + } + +} +#if OPT_CACHE_BLOCKING_L1 //new intrinsic kernels +static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0) + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0) + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done + +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} +#else //rel 1.0 intrisic kernels (NOT OPT_CACHE_BLOCKING_L1) +static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + __m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + __m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); + mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); + mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); + mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); + mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); + mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); + mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); + mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); + mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); + + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + //__m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0) + mat_b_col[0] = mat_b_rearr[0][0]; + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): already done + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + //__m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); + mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); + mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); + mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); + mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); + + //(Row0) + mat_b_col[0] = mat_b_rearr[0][0]; + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); + mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); + mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); + mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); + mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); + + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): already done + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} +#endif //OPT_CACHE_BLOCKING_L1 + +//////////////////////////// AutX=B /////////////////////// +static void trsm_AutXB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += 8; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ +#endif + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_AutXB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += 8; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_AutXB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int i, i1, i2, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + + //(Row0) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ +#endif + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done + +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + //float ones = 1.0; + int i, i1, i2, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done + +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} +#endif diff --git a/test/Makefile b/test/Makefile index 799900b58..27d3eaa2c 100644 --- a/test/Makefile +++ b/test/Makefile @@ -5,7 +5,7 @@ # libraries. # # Copyright (C) 2014, The University of Texas at Austin -# Copyright (C) 2017, Advanced Micro Devices, Inc. +# Copyright (C) 2017 - 2018, Advanced Micro Devices, Inc. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are From 7fe44748383071f1cbbc77d904f4ae5538e13065 Mon Sep 17 00:00:00 2001 From: Kiran Varaganti Date: Wed, 6 Mar 2019 16:23:31 +0530 Subject: [PATCH 26/53] Disabled BLIS_ENABLE_ZEN_BLOCK_SIZES in bli_family_zen.h for ROME tuning Change-Id: Iec47fcf51f4d4396afef1ce3958e58cf02c59a57 --- config/zen/bli_family_zen.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/zen/bli_family_zen.h b/config/zen/bli_family_zen.h index d69f5270a..12242727e 100644 --- a/config/zen/bli_family_zen.h +++ b/config/zen/bli_family_zen.h @@ -42,7 +42,7 @@ #define BLIS_THREAD_MAX_IR 1 #define BLIS_THREAD_MAX_JR 1 -#define BLIS_ENABLE_ZEN_BLOCK_SIZES +//#define BLIS_ENABLE_ZEN_BLOCK_SIZES #define BLIS_ENABLE_SMALL_MATRIX #define BLIS_ENABLE_SMALL_MATRIX_TRSM From 3a929a3d0ba0353159a6d4cd188f01b7a390ccfc Mon Sep 17 00:00:00 2001 From: Kiran Varaganti Date: Mon, 18 Mar 2019 10:51:41 +0530 Subject: [PATCH 27/53] Fixed code merging: bli_gemm_small.c - missed conditional checks for L!=0 && K!=0. Now they are added. This fix is done to pass blastest Change-Id: Idc9c9a04d2015a68a19553c437ecaf8f1584026c --- kernels/zen/3/bli_gemm_small.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/kernels/zen/3/bli_gemm_small.c b/kernels/zen/3/bli_gemm_small.c index 776c5a0fe..52b5ebd54 100644 --- a/kernels/zen/3/bli_gemm_small.c +++ b/kernels/zen/3/bli_gemm_small.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2018, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -176,10 +176,11 @@ static err_t bli_sgemm_small int M = bli_obj_length( c ); // number of rows of Matrix C int N = bli_obj_width( c ); // number of columns of Matrix C int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + int L = M * N; // printf("alpha_cast = %f beta_cast = %f [ Trans = %d %d], [stride = %d %d %d] [m,n,k = %d %d %d]\n",*alpha_cast,*beta_cast, bli_obj_has_trans( a ), bli_obj_has_trans( b ), lda, ldb,ldc, M,N,K); - if (((M * N) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES)) - || ((M < BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < BLIS_SMALL_K_RECT_MATRIX_THRES))) + if ((((L) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES)) + || ((M < BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) { int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. @@ -1577,11 +1578,12 @@ static err_t bli_dgemm_small int M = bli_obj_length( c ); // number of rows of Matrix C int N = bli_obj_width( c ); // number of columns of Matrix C int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + int L = M * N; // If alpha is zero, scale by beta and return. // printf("alpha_cast = %f beta_cast = %f [ Trans = %d %d], [stride = %d %d %d] [m,n,k = %d %d %d]\n",*alpha_cast,*beta_cast, bli_obj_has_trans( a ), bli_obj_has_trans( b ), lda, ldb,ldc, M,N,K); - if (((M * N) < (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES)) - || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) + if ((((L) < (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES)) + || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) { int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. From 20153cd4b594bc34f860c381ec18de3a6cc743c7 Mon Sep 17 00:00:00 2001 From: Kiran Varaganti Date: Thu, 21 Mar 2019 16:23:53 +0530 Subject: [PATCH 28/53] Modified test_gemm.c file in test folder A Macro 'FILE_IN_OUT" is defined to read input parameters from a csv file. Format for input file: Each line defines a gemm problem with following parameters: m k n cs_a cs_b cs_c The operation always implemented is C = C - A*B and column-major format. When macro is disabled - it reverts back to original implementation. Usage: ./test_gemm_.x input.csv output.csv GEMM is called through BLAS interface For BLIS - the test application also prints either 'S' indicating small gemm routine or 'N' - conventional BLIS gemm for MKL/OpenBLAS - ignore this character Change-Id: I0924ef2c1f7bdea48d4cdb230b888e2af2c86a36 --- test/test_gemm.c | 116 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 100 insertions(+), 16 deletions(-) diff --git a/test/test_gemm.c b/test/test_gemm.c index 5d6b6aa9a..a587e6765 100644 --- a/test/test_gemm.c +++ b/test/test_gemm.c @@ -36,6 +36,9 @@ #include "blis.h" +#define FILE_IN_OUT // File based input matrix dimensions + + //#define PRINT int main( int argc, char** argv ) @@ -58,11 +61,19 @@ int main( int argc, char** argv ) double dtime_save; double gflops; + +#ifdef FILE_IN_OUT + FILE* fin = NULL; + FILE* fout = NULL; + char gemm = 's'; + +#endif + //bli_init(); //bli_error_checking_level_set( BLIS_NO_ERROR_CHECKING ); - n_repeats = 3; + n_repeats = 10; #ifndef PRINT p_begin = 200; @@ -96,20 +107,53 @@ int main( int argc, char** argv ) bli_param_map_blis_to_netlib_trans( transa, &f77_transa ); bli_param_map_blis_to_netlib_trans( transb, &f77_transb ); - // Begin with initializing the last entry to zero so that - // matlab allocates space for the entire array once up-front. - for ( p = p_begin; p + p_inc <= p_end; p += p_inc ) ; -#ifdef BLIS - printf( "data_gemm_blis" ); -#else - printf( "data_gemm_%s", BLAS ); -#endif - printf( "( %2lu, 1:4 ) = [ %4lu %4lu %4lu %7.2f ];\n", - ( unsigned long )(p - p_begin + 1)/p_inc + 1, - ( unsigned long )0, - ( unsigned long )0, - ( unsigned long )0, 0.0 ); +#ifdef FILE_IN_OUT + if (argc < 3) + { + printf("Usage: ./test_gemm_XX.x input.csv output.csv\n"); + exit(1); + } + fin = fopen(argv[1], "r"); + if (fin == NULL) + { + printf("Error opening the file %s\n", argv[1]); + exit(1); + } + fout = fopen(argv[2], "w"); + if (fout == NULL) + { + printf("Error opening output file %s\n", argv[2]); + exit(1); + } + fprintf(fout, "m\t k\t n\t cs_a\t cs_b\t cs_c\t gflops\t GEMM_Algo\n"); + + + printf("~~~~~~~~~~_BLAS\t m\t k\t n\t cs_a\t cs_b\t cs_c \t gflops\t GEMM_Algo\n"); + + inc_t cs_a; + inc_t cs_b; + inc_t cs_c; + + while (fscanf(fin, "%ld %ld %ld %ld %ld %ld\n", &m, &k, &n, &cs_a, &cs_b, &cs_c) == 6) + { + if ((m > cs_a) || (k > cs_b) || (m > cs_c)) continue; // leading dimension should be greater than number of rows + + bli_obj_create( dt, 1, 1, 0, 0, &alpha); + bli_obj_create( dt, 1, 1, 0, 0, &beta ); + + bli_obj_create( dt, m, k, 1, cs_a, &a ); + bli_obj_create( dt, k, n, 1, cs_b, &b ); + bli_obj_create( dt, m, n, 1, cs_c, &c ); + bli_obj_create( dt, m, n, 1, cs_c, &c_save ); + + bli_obj_set_conjtrans( transa, &a); + bli_obj_set_conjtrans( transb, &b); + + bli_setsc( 0.0, -1, &alpha ); + bli_setsc( 0.0, 1, &beta ); + +#else for ( p = p_begin; p <= p_end; p += p_inc ) { if ( m_input < 0 ) m = p * ( dim_t )abs(m_input); @@ -136,7 +180,7 @@ int main( int argc, char** argv ) bli_setsc( (0.9/1.0), 0.2, &alpha ); bli_setsc( -(1.1/1.0), 0.3, &beta ); - +#endif bli_copym( &c, &c_save ); @@ -156,7 +200,7 @@ int main( int argc, char** argv ) bli_printm( "c", &c, "%4.1f", "" ); #endif -#ifdef BLIS +#if 0 //def BLIS bli_gemm( &alpha, &a, @@ -179,6 +223,7 @@ int main( int argc, char** argv ) float* bp = bli_obj_buffer( &b ); float* betap = bli_obj_buffer( &beta ); float* cp = bli_obj_buffer( &c ); + sgemm_( &f77_transa, &f77_transb, @@ -286,12 +331,46 @@ int main( int argc, char** argv ) #else printf( "data_gemm_%s", BLAS ); #endif + + +#ifdef FILE_IN_OUT + + if ( bli_is_double( dt ) ) { + + if (((m * n) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES/4)) || ((m < (BLIS_SMALL_M_RECT_MATRIX_THRES/2) ) && (k < (BLIS_SMALL_K_RECT_MATRIX_THRES/2) ))) + gemm = 'S'; // small gemm + else gemm = 'N'; // Normal blis gemm + + } + else if (bli_is_float( dt )) { + if (((m * n) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES)) || ((m < BLIS_SMALL_M_RECT_MATRIX_THRES) && (k < BLIS_SMALL_K_RECT_MATRIX_THRES))) + gemm = 'S'; // small gemm + else gemm = 'N'; // normal blis gemm + } + + + + printf("%6lu \t %4lu \t %4lu \t %4lu \t %4lu \t %4lu \t %6.3f \t %c\n", \ + ( unsigned long )m, + ( unsigned long )k, + ( unsigned long )n, (unsigned long)cs_a, (unsigned long)cs_b, (unsigned long)cs_c, gflops, gemm ); + + + fprintf(fout, "%6lu \t %4lu \t %4lu \t %4lu \t %4lu \t %4lu \t %6.3f \t %c\n", \ + ( unsigned long )m, + ( unsigned long )k, + ( unsigned long )n, (unsigned long)cs_a, (unsigned long)cs_b, (unsigned long)cs_c, gflops, gemm ); + fflush(fout); + +#else printf( "( %2lu, 1:4 ) = [ %4lu %4lu %4lu %7.2f ];\n", ( unsigned long )(p - p_begin + 1)/p_inc + 1, ( unsigned long )m, ( unsigned long )k, ( unsigned long )n, gflops ); + +#endif bli_obj_free( &alpha ); bli_obj_free( &beta ); @@ -303,6 +382,11 @@ int main( int argc, char** argv ) //bli_finalize(); +#ifdef FILE_IN_OUT + fclose(fin); + fclose(fout); +#endif + return 0; } From 6805db45e343d83d1adaf9157cf0b841653e9ede Mon Sep 17 00:00:00 2001 From: Kiran Varaganti Date: Fri, 22 Mar 2019 12:55:35 +0530 Subject: [PATCH 29/53] Corrected setting alpha & beta values- alpha = -1 and beta = 1 - bli_setc(-1.0, 0, &alpha) should be used rather than bli_setc(0.0, -1.0, &alpha). This corrected now Change-Id: Ic1102dfd6b50ccf212386a1211c6f31e8d987ef9 --- test/test_gemm.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/test_gemm.c b/test/test_gemm.c index a587e6765..b306ef519 100644 --- a/test/test_gemm.c +++ b/test/test_gemm.c @@ -150,8 +150,13 @@ int main( int argc, char** argv ) bli_obj_set_conjtrans( transa, &a); bli_obj_set_conjtrans( transb, &b); - bli_setsc( 0.0, -1, &alpha ); - bli_setsc( 0.0, 1, &beta ); + //bli_setsc( 0.0, -1, &alpha ); + //bli_setsc( 0.0, 1, &beta ); + + bli_setsc( -1, 0.0, &alpha ); + bli_setsc( 1, 0.0, &beta ); + + printf("%1.1f %1.1f\n", *((double *)bli_obj_buffer_for_const(BLIS_FLOAT, &alpha)), *((double *)bli_obj_buffer_for_const(BLIS_FLOAT, &beta))); #else for ( p = p_begin; p <= p_end; p += p_inc ) From 53842c7e7d530cb2d5609d6d124ae350fc345c32 Mon Sep 17 00:00:00 2001 From: Kiran Varaganti Date: Fri, 22 Mar 2019 13:57:14 +0530 Subject: [PATCH 30/53] Removed printing alpha and beta values Change-Id: I49102db510311a30f6a936f9d843f35838f50d23 --- test/test_gemm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_gemm.c b/test/test_gemm.c index b306ef519..823c95e0e 100644 --- a/test/test_gemm.c +++ b/test/test_gemm.c @@ -156,7 +156,7 @@ int main( int argc, char** argv ) bli_setsc( -1, 0.0, &alpha ); bli_setsc( 1, 0.0, &beta ); - printf("%1.1f %1.1f\n", *((double *)bli_obj_buffer_for_const(BLIS_FLOAT, &alpha)), *((double *)bli_obj_buffer_for_const(BLIS_FLOAT, &beta))); + // printf("%1.1f %1.1f\n", *((double *)bli_obj_buffer_for_const(BLIS_FLOAT, &alpha)), *((double *)bli_obj_buffer_for_const(BLIS_FLOAT, &beta))); #else for ( p = p_begin; p <= p_end; p += p_inc ) From 9d76688ad90014a11ddc0c2f27253d62806216b1 Mon Sep 17 00:00:00 2001 From: kdevraje Date: Thu, 11 Apr 2019 10:22:48 +0530 Subject: [PATCH 31/53] Fix for single rank crash with HPL application. When computing offset of C buffer, as integer variables are used for a row and column index, the intermediate result value overflows and a negative value gets added to the buffer, when the negative value is too large it would index the buffer out of the range resulting in segmentation fault. Although the crash is a result of dgemm kernel, added similar code in sgemm kernel also. Change-Id: I171119b0ec0dfbd8e63f1fcd6609a94384aabd27 --- kernels/zen/3/bli_gemm_small.c | 82 +++++++++++++++++----------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/kernels/zen/3/bli_gemm_small.c b/kernels/zen/3/bli_gemm_small.c index 52b5ebd54..510dab010 100644 --- a/kernels/zen/3/bli_gemm_small.c +++ b/kernels/zen/3/bli_gemm_small.c @@ -173,29 +173,29 @@ static err_t bli_sgemm_small ) { - int M = bli_obj_length( c ); // number of rows of Matrix C - int N = bli_obj_width( c ); // number of columns of Matrix C - int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . - int L = M * N; + gint_t M = bli_obj_length( c ); // number of rows of Matrix C + gint_t N = bli_obj_width( c ); // number of columns of Matrix C + gint_t K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + gint_t L = M * N; // printf("alpha_cast = %f beta_cast = %f [ Trans = %d %d], [stride = %d %d %d] [m,n,k = %d %d %d]\n",*alpha_cast,*beta_cast, bli_obj_has_trans( a ), bli_obj_has_trans( b ), lda, ldb,ldc, M,N,K); if ((((L) < (BLIS_SMALL_MATRIX_THRES * BLIS_SMALL_MATRIX_THRES)) || ((M < BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) { - int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. - int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. - int ldc = bli_obj_col_stride( c ); // column stride of matrix C - int row_idx, col_idx, k; + guint_t lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + guint_t ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + guint_t ldc = bli_obj_col_stride( c ); // column stride of matrix C + guint_t row_idx, col_idx, k; float *A = a->buffer; // pointer to elements of Matrix A float *B = b->buffer; // pointer to elements of Matrix B float *C = c->buffer; // pointer to elements of Matrix C float *tA = A, *tB = B, *tC = C;//, *tA_pack; float *tA_packed; // temprorary pointer to hold packed A memory pointer - int row_idx_packed; //packed A memory row index - int lda_packed; //lda of packed A - int col_idx_start; //starting index after A matrix is packed. + guint_t row_idx_packed; //packed A memory row index + guint_t lda_packed; //lda of packed A + guint_t col_idx_start; //starting index after A matrix is packed. dim_t tb_inc_row = 1; // row stride of matrix B dim_t tb_inc_col = ldb; // column stride of matrix B __m256 ymm4, ymm5, ymm6, ymm7; @@ -203,13 +203,13 @@ static err_t bli_sgemm_small __m256 ymm12, ymm13, ymm14, ymm15; __m256 ymm0, ymm1, ymm2, ymm3; - int n_remainder; // If the N is non multiple of 3.(N%3) - int m_remainder; // If the M is non multiple of 32.(M%32) + gint_t n_remainder; // If the N is non multiple of 3.(N%3) + gint_t m_remainder; // If the M is non multiple of 32.(M%32) float *alpha_cast, *beta_cast; // alpha, beta multiples alpha_cast = (alpha->buffer); beta_cast = (beta->buffer); - int required_packing_A = 1; + gint_t required_packing_A = 1; // when N is equal to 1 call GEMV instead of GEMM if (N == 1) @@ -1575,10 +1575,10 @@ static err_t bli_dgemm_small ) { - int M = bli_obj_length( c ); // number of rows of Matrix C - int N = bli_obj_width( c ); // number of columns of Matrix C - int K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . - int L = M * N; + gint_t M = bli_obj_length( c ); // number of rows of Matrix C + gint_t N = bli_obj_width( c ); // number of columns of Matrix C + gint_t K = bli_obj_width( a ); // number of columns of OP(A), will be updated if OP(A) is Transpose(A) . + gint_t L = M * N; // If alpha is zero, scale by beta and return. // printf("alpha_cast = %f beta_cast = %f [ Trans = %d %d], [stride = %d %d %d] [m,n,k = %d %d %d]\n",*alpha_cast,*beta_cast, bli_obj_has_trans( a ), bli_obj_has_trans( b ), lda, ldb,ldc, M,N,K); @@ -1586,19 +1586,19 @@ static err_t bli_dgemm_small || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) { - int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. - int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. - int ldc = bli_obj_col_stride( c ); // column stride of matrix C - int row_idx, col_idx, k; + guint_t lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + guint_t ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + guint_t ldc = bli_obj_col_stride( c ); // column stride of matrix C + guint_t row_idx, col_idx, k; double *A = a->buffer; // pointer to elements of Matrix A double *B = b->buffer; // pointer to elements of Matrix B double *C = c->buffer; // pointer to elements of Matrix C double *tA = A, *tB = B, *tC = C;//, *tA_pack; double *tA_packed; // temprorary pointer to hold packed A memory pointer - int row_idx_packed; //packed A memory row index - int lda_packed; //lda of packed A - int col_idx_start; //starting index after A matrix is packed. + guint_t row_idx_packed; //packed A memory row index + guint_t lda_packed; //lda of packed A + guint_t col_idx_start; //starting index after A matrix is packed. dim_t tb_inc_row = 1; // row stride of matrix B dim_t tb_inc_col = ldb; // column stride of matrix B __m256d ymm4, ymm5, ymm6, ymm7; @@ -1606,13 +1606,13 @@ static err_t bli_dgemm_small __m256d ymm12, ymm13, ymm14, ymm15; __m256d ymm0, ymm1, ymm2, ymm3; - int n_remainder; // If the N is non multiple of 3.(N%3) - int m_remainder; // If the M is non multiple of 16.(M%16) + gint_t n_remainder; // If the N is non multiple of 3.(N%3) + gint_t m_remainder; // If the M is non multiple of 16.(M%16) double *alpha_cast, *beta_cast; // alpha, beta multiples alpha_cast = (alpha->buffer); beta_cast = (beta->buffer); - int required_packing_A = 1; + gint_t required_packing_A = 1; // when N is equal to 1 call GEMV instead of GEMM if (N == 1) @@ -2978,12 +2978,12 @@ static err_t bli_sgemm_small_atbn cntl_t* cntl ) { - int M = bli_obj_length( c ); // number of rows of Matrix C - int N = bli_obj_width( c ); // number of columns of Matrix C - int K = bli_obj_length( b ); // number of rows of Matrix B - int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. - int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. - int ldc = bli_obj_col_stride( c ); // column stride of matrix C + gint_t M = bli_obj_length( c ); // number of rows of Matrix C + gint_t N = bli_obj_width( c ); // number of columns of Matrix C + gint_t K = bli_obj_length( b ); // number of rows of Matrix B + guint_t lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + guint_t ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + guint_t ldc = bli_obj_col_stride( c ); // column stride of matrix C int row_idx = 0, col_idx = 0, k; float *A = a->buffer; // pointer to matrix A elements, stored in row major format float *B = b->buffer; // pointer to matrix B elements, stored in column major format @@ -3370,13 +3370,13 @@ static err_t bli_dgemm_small_atbn cntl_t* cntl ) { - int M = bli_obj_length( c ); // number of rows of Matrix C - int N = bli_obj_width( c ); // number of columns of Matrix C - int K = bli_obj_length( b ); // number of rows of Matrix B - int lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. - int ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. - int ldc = bli_obj_col_stride( c ); // column stride of matrix C - int row_idx = 0, col_idx = 0, k; + gint_t M = bli_obj_length( c ); // number of rows of Matrix C + gint_t N = bli_obj_width( c ); // number of columns of Matrix C + gint_t K = bli_obj_length( b ); // number of rows of Matrix B + guint_t lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. + guint_t ldb = bli_obj_col_stride( b ); // column stride of matrix OP(B), where OP(B) is Transpose(B) if transB enabled. + guint_t ldc = bli_obj_col_stride( c ); // column stride of matrix C + guint_t row_idx = 0, col_idx = 0, k; double *A = a->buffer; // pointer to matrix A elements, stored in row major format double *B = b->buffer; // pointer to matrix B elements, stored in column major format double *C = c->buffer; // pointer to matrix C elements, stored in column major format From ca4b33c001f9e959c43b95a9a23f9df5adec7adf Mon Sep 17 00:00:00 2001 From: Kiran Varaganti Date: Wed, 24 Apr 2019 15:02:39 +0530 Subject: [PATCH 32/53] Added compiler option (-mno-avx256-split-unaligned-store) in the file config/zen/make_defs.mk to improve performance of intrinsic codes, this flag ensures compiler generates 256-bit stores for the equivalent intrinsics code. Change-Id: I8f8cd81a3604869df18d38bc42097a04f178d324 --- config/zen/make_defs.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/zen/make_defs.mk b/config/zen/make_defs.mk index a485af27b..5ef257107 100644 --- a/config/zen/make_defs.mk +++ b/config/zen/make_defs.mk @@ -64,7 +64,7 @@ endif CKOPTFLAGS := $(COPTFLAGS) ifeq ($(CC_VENDOR),gcc) # gcc 6.0 (clang 4.0) or later: -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 +CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-avx256-split-unaligned-store # gcc 4.9 (clang 3.5) or later: # possibly add zen-specific instructions: -mclzero -madx -mrdseed -mmwaitx -msha -mxsavec -mxsaves -mclflushopt -mpopcnt #CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp From ab0818af80f7f683080873f3fa24734b65267df2 Mon Sep 17 00:00:00 2001 From: sraut Date: Wed, 3 Oct 2018 15:30:33 +0530 Subject: [PATCH 33/53] Review comments incorporated for small TRSM. Change-Id: Ia64b7b2c0375cc501c2cb0be8a1af93111808cd9 From c72ae27adee4726679ee004d02c972582b5285b4 Mon Sep 17 00:00:00 2001 From: Nisanth M P Date: Mon, 19 Mar 2018 12:49:26 +0530 Subject: [PATCH 34/53] Re-enabling the small matrix gemm optimization for target zen Change-Id: I13872784586984634d728cd99a00f71c3f904395 From 17b878b66d917d50b6fe23721d8579e826cb3e8c Mon Sep 17 00:00:00 2001 From: kdevraje Date: Wed, 22 May 2019 14:02:53 +0530 Subject: [PATCH 35/53] adding license same as in ut-austin-amd-branch Change-Id: I6790768d2bf5d42369d304ef93e34701f95fbaff --- LICENSE | 4 +++- config/zen/old/bli_kernel.h | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/LICENSE b/LICENSE index 559e3627d..1c6845375 100644 --- a/LICENSE +++ b/LICENSE @@ -4,6 +4,7 @@ NOTE: Portions of this project's code are copyrighted by while other portions are copyrighted by + Hewlett Packard Enterprise Development LP Advanced Micro Devices, Inc. with some overlap. Please see file-level license headers for file-specific @@ -12,8 +13,9 @@ copyright info. All parties provide their portions of the code under the --- +Copyright (C) 2018, The University of Texas at Austin +Copyright (C) 2016, Hewlett Packard Enterprise Development LP Copyright (C) 2018, Advanced Micro Devices, Inc. -Copyright (C) 2014, The University of Texas at Austin Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/config/zen/old/bli_kernel.h b/config/zen/old/bli_kernel.h index 6957f059d..68b9e88e0 100644 --- a/config/zen/old/bli_kernel.h +++ b/config/zen/old/bli_kernel.h @@ -1,10 +1,11 @@ /* - BLIS + BLIS An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2018, Advanced Micro Devices, Inc + Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are From a23f92594cf3d530e5794307fe97afc877d853b7 Mon Sep 17 00:00:00 2001 From: Kiran Varaganti Date: Mon, 20 May 2019 10:48:06 +0530 Subject: [PATCH 36/53] config_registry: New AMD zen2 architecture configuration added. frame/base/bli_arch.c: #ifdef BLIS_FAMILY_ZEN2 id = BLIS_ARCH_ZEN2; #endif added. zen2 is added in config_name[BLIS_NUM_ARCHS] frame/base/bli_cpuid.c : #ifdef BLIS_CONFIG_ZEN2 if ( bli_cpuid_is_zen2( family, model, features ) ) return BLIS_ARCH_ZEN2; #endif, defined new function bool bli_cpuid_is_zen2(...). frame/base/bli_cpuid.h : declared bli_cpuid_is_zen2(..). frame/base/bli_gks.c : #ifdef BLIS_CONFIG_ZEN2 bli_gks_register_cntx(BLIS_ARCH_ZEN2, bli_cntx_init_zen2, bli_cntx_init_zen2_ref, bli_cntx_init_zen2_ind); #endif frame/include/bli_arch_config.h : #ifdef BLIS_CONFIG_ZEN2 CNTX_INIT_PROTS(zen2) #endif #ifdef BLIS_FAMILY_ZEN2 #include "bli_family_zen2.h" #endif frame/include/bli_type_defs.h : added BLIS_ARCH_ZEN2 in arch_t enum. BLIS_NUM_ARCHS 20 Change-Id: I2a2d9b7266673e78a4f8543b1bfb5425b0aa7866 --- config/zen2/bli_cntx_init_zen2.c | 136 +++++++++++++++++++++++++++++++ config/zen2/bli_family_zen2.h | 60 ++++++++++++++ config/zen2/make_defs.mk | 87 ++++++++++++++++++++ config_registry | 3 +- frame/base/bli_arch.c | 6 +- frame/base/bli_cpuid.c | 34 +++++++- frame/base/bli_cpuid.h | 2 + frame/base/bli_gks.c | 7 +- frame/include/bli_arch_config.h | 8 +- frame/include/bli_type_defs.h | 5 +- 10 files changed, 341 insertions(+), 7 deletions(-) create mode 100644 config/zen2/bli_cntx_init_zen2.c create mode 100644 config/zen2/bli_family_zen2.h create mode 100644 config/zen2/make_defs.mk diff --git a/config/zen2/bli_cntx_init_zen2.c b/config/zen2/bli_cntx_init_zen2.c new file mode 100644 index 000000000..20bdcf90b --- /dev/null +++ b/config/zen2/bli_cntx_init_zen2.c @@ -0,0 +1,136 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2019, Advanced Micro Devices, Inc. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" + +void bli_cntx_init_zen2( cntx_t* cntx ) +{ + blksz_t blkszs[ BLIS_NUM_BLKSZS ]; + + // Set default kernel blocksizes and functions. + bli_cntx_init_zen2_ref( cntx ); + + // ------------------------------------------------------------------------- + + // Update the context with optimized native gemm micro-kernels and + // their storage preferences. + bli_cntx_set_l3_nat_ukrs + ( + 8, + // gemm + BLIS_GEMM_UKR, BLIS_FLOAT, bli_sgemm_haswell_asm_6x16, TRUE, + BLIS_GEMM_UKR, BLIS_DOUBLE, bli_dgemm_haswell_asm_6x8, TRUE, + BLIS_GEMM_UKR, BLIS_SCOMPLEX, bli_cgemm_haswell_asm_3x8, TRUE, + BLIS_GEMM_UKR, BLIS_DCOMPLEX, bli_zgemm_haswell_asm_3x4, TRUE, + // gemmtrsm_l + BLIS_GEMMTRSM_L_UKR, BLIS_FLOAT, bli_sgemmtrsm_l_haswell_asm_6x16, TRUE, + BLIS_GEMMTRSM_L_UKR, BLIS_DOUBLE, bli_dgemmtrsm_l_haswell_asm_6x8, TRUE, + // gemmtrsm_u + BLIS_GEMMTRSM_U_UKR, BLIS_FLOAT, bli_sgemmtrsm_u_haswell_asm_6x16, TRUE, + BLIS_GEMMTRSM_U_UKR, BLIS_DOUBLE, bli_dgemmtrsm_u_haswell_asm_6x8, TRUE, + cntx + ); + + // Update the context with optimized level-1f kernels. + bli_cntx_set_l1f_kers + ( + 4, + // axpyf + BLIS_AXPYF_KER, BLIS_FLOAT, bli_saxpyf_zen_int_8, + BLIS_AXPYF_KER, BLIS_DOUBLE, bli_daxpyf_zen_int_8, + // dotxf + BLIS_DOTXF_KER, BLIS_FLOAT, bli_sdotxf_zen_int_8, + BLIS_DOTXF_KER, BLIS_DOUBLE, bli_ddotxf_zen_int_8, + cntx + ); + + // Update the context with optimized level-1v kernels. + bli_cntx_set_l1v_kers + ( + 10, + // amaxv + BLIS_AMAXV_KER, BLIS_FLOAT, bli_samaxv_zen_int, + BLIS_AMAXV_KER, BLIS_DOUBLE, bli_damaxv_zen_int, + // axpyv + + BLIS_AXPYV_KER, BLIS_FLOAT, bli_saxpyv_zen_int10, + BLIS_AXPYV_KER, BLIS_DOUBLE, bli_daxpyv_zen_int10, + + // dotv + BLIS_DOTV_KER, BLIS_FLOAT, bli_sdotv_zen_int, + BLIS_DOTV_KER, BLIS_DOUBLE, bli_ddotv_zen_int, + // dotxv + BLIS_DOTXV_KER, BLIS_FLOAT, bli_sdotxv_zen_int, + BLIS_DOTXV_KER, BLIS_DOUBLE, bli_ddotxv_zen_int, + // scalv + + BLIS_SCALV_KER, BLIS_FLOAT, bli_sscalv_zen_int10, + BLIS_SCALV_KER, BLIS_DOUBLE, bli_dscalv_zen_int10, + + cntx + ); + + // Initialize level-3 blocksize objects with architecture-specific values. + // s d c z + bli_blksz_init_easy( &blkszs[ BLIS_MR ], 6, 6, 3, 3 ); + bli_blksz_init_easy( &blkszs[ BLIS_NR ], 16, 8, 8, 4 ); + + + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); + + + bli_blksz_init_easy( &blkszs[ BLIS_AF ], 8, 8, -1, -1 ); + bli_blksz_init_easy( &blkszs[ BLIS_DF ], 8, 8, -1, -1 ); + + // Update the context with the current architecture's register and cache + // blocksizes (and multiples) for native execution. + bli_cntx_set_blkszs + ( + BLIS_NAT, 7, + // level-3 + BLIS_NC, &blkszs[ BLIS_NC ], BLIS_NR, + BLIS_KC, &blkszs[ BLIS_KC ], BLIS_KR, + BLIS_MC, &blkszs[ BLIS_MC ], BLIS_MR, + BLIS_NR, &blkszs[ BLIS_NR ], BLIS_NR, + BLIS_MR, &blkszs[ BLIS_MR ], BLIS_MR, + // level-1f + BLIS_AF, &blkszs[ BLIS_AF ], BLIS_AF, + BLIS_DF, &blkszs[ BLIS_DF ], BLIS_DF, + cntx + ); +} + diff --git a/config/zen2/bli_family_zen2.h b/config/zen2/bli_family_zen2.h new file mode 100644 index 000000000..a41c0222d --- /dev/null +++ b/config/zen2/bli_family_zen2.h @@ -0,0 +1,60 @@ +/* + + BLIS + An object-based framework for developing high-performance BLAS-like + libraries. + + Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2019, Advanced Micro Devices, Inc + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + - Neither the name(s) of the copyright holder(s) nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +//#ifndef BLIS_FAMILY_H +//#define BLIS_FAMILY_H + +// By default, it is effective to parallelize the outer loops. +// Setting these macros to 1 will force JR and IR inner loops +// to be not paralleized. +#define BLIS_THREAD_MAX_IR 1 +#define BLIS_THREAD_MAX_JR 1 + + +#define BLIS_ENABLE_SMALL_MATRIX +#define BLIS_ENABLE_SMALL_MATRIX_TRSM + + +// This will select the threshold below which small matrix code will be called. +#define BLIS_SMALL_MATRIX_THRES 700 +#define BLIS_SMALL_M_RECT_MATRIX_THRES 160 +#define BLIS_SMALL_K_RECT_MATRIX_THRES 128 + +#define BLIS_SMALL_MATRIX_THRES_TRSM 32768 //128(128+128) => m*(m+n) +#define BLIS_SMALL_MATRIX_A_THRES_TRSM 128 +#define BLIS_SMALL_MATRIX_A_THRES_M_SYRK 96 +#define BLIS_SMALL_MATRIX_A_THRES_N_SYRK 128 + + diff --git a/config/zen2/make_defs.mk b/config/zen2/make_defs.mk new file mode 100644 index 000000000..dae064432 --- /dev/null +++ b/config/zen2/make_defs.mk @@ -0,0 +1,87 @@ +# +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2014, The University of Texas at Austin +# Copyright (C) 2019, Advanced Micro Devices, Inc. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name(s) of the copyright holder(s) nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + + +# Declare the name of the current configuration and add it to the +# running list of configurations included by common.mk. +THIS_CONFIG := zen2 +#CONFIGS_INCL += $(THIS_CONFIG) + +# +# --- Determine the C compiler and related flags --- +# + +# NOTE: The build system will append these variables with various +# general-purpose/configuration-agnostic flags in common.mk. You +# may specify additional flags here as needed. +CPPROCFLAGS := +CMISCFLAGS := +CPICFLAGS := +CWARNFLAGS := + +ifneq ($(DEBUG_TYPE),off) +CDBGFLAGS := -g +endif + +ifeq ($(DEBUG_TYPE),noopt) +COPTFLAGS := -O0 +else +COPTFLAGS := -O3 -fomit-frame-pointer +endif + +# Flags specific to optimized kernels. +CKOPTFLAGS := $(COPTFLAGS) +ifeq ($(CC_VENDOR),gcc) +# gcc 9.0 (clang ?) or later: +CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver2 +# gcc 4.9 (clang 3.5) or later: +# possibly add zen-specific instructions: -mclzero -madx -mrdseed -mmwaitx -msha -mxsavec -mxsaves -mclflushopt -mpopcnt +#CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +else +ifeq ($(CC_VENDOR),clang) +CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver2 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +else +$(error gcc or clang are required for this configuration.) +endif +endif + +# Flags specific to reference kernels. +CROPTFLAGS := $(CKOPTFLAGS) +CRVECFLAGS := $(CKVECFLAGS) + +# Store all of the variables here to new variables containing the +# configuration name. +$(eval $(call store-make-defs,$(THIS_CONFIG))) + diff --git a/config_registry b/config_registry index 5f66f0cb5..2744fa4f0 100644 --- a/config_registry +++ b/config_registry @@ -10,7 +10,7 @@ # Processor families. x86_64: intel64 amd64 intel64: skx knl haswell sandybridge penryn generic -amd64: zen excavator steamroller piledriver bulldozer generic +amd64: zen2 zen excavator steamroller piledriver bulldozer generic # NOTE: ARM families will remain disabled until runtime hardware detection # logic is added to BLIS. #arm64: cortexa57 generic @@ -24,6 +24,7 @@ sandybridge: sandybridge penryn: penryn # AMD architectures. +zen2: zen2/zen2/zen/haswell zen: zen/zen/haswell excavator: excavator/piledriver steamroller: steamroller/piledriver diff --git a/frame/base/bli_arch.c b/frame/base/bli_arch.c index 67ee5e806..0246094a1 100644 --- a/frame/base/bli_arch.c +++ b/frame/base/bli_arch.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018-2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -103,6 +103,9 @@ void bli_arch_set_id( void ) #endif // AMD microarchitectures. +#ifdef BLIS_FAMILY_ZEN2 + id = BLIS_ARCH_ZEN2; +#endif #ifdef BLIS_FAMILY_ZEN id = BLIS_ARCH_ZEN; #endif @@ -168,6 +171,7 @@ static char* config_name[ BLIS_NUM_ARCHS ] = "sandybridge", "penryn", + "zen2", "zen", "excavator", "steamroller", diff --git a/frame/base/bli_cpuid.c b/frame/base/bli_cpuid.c index 6b17a1dac..e4feb7249 100644 --- a/frame/base/bli_cpuid.c +++ b/frame/base/bli_cpuid.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018-2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -99,6 +99,10 @@ arch_t bli_cpuid_query_id( void ) // Check for each AMD configuration that is enabled, check for that // microarchitecture. We check from most recent to most dated. +#ifdef BLIS_CONFIG_ZEN2 + if ( bli_cpuid_is_zen2( family, model, features ) ) + return BLIS_ARCH_ZEN2; +#endif #ifdef BLIS_CONFIG_ZEN if ( bli_cpuid_is_zen( family, model, features ) ) return BLIS_ARCH_ZEN; @@ -227,6 +231,34 @@ bool_t bli_cpuid_is_penryn // ----------------------------------------------------------------------------- +bool_t bli_cpuid_is_zen2 + ( + uint32_t family, + uint32_t model, + uint32_t features + ) +{ + // Check for expected CPU features. + const uint32_t expected = FEATURE_AVX | + FEATURE_FMA3 | + FEATURE_AVX2; + + if ( !bli_cpuid_has_features( features, expected ) ) return FALSE; + + // All Zen cores have a family of 0x17. + if ( family != 0x17 ) return FALSE; + + // Finally, check for specific models: + // - 0x00-0xff (THIS NEEDS UPDATING) + const bool_t is_arch + = + ( 0x00 <= model && model <= 0xff ); + + if ( !is_arch ) return FALSE; + + return TRUE; +} + bool_t bli_cpuid_is_zen ( uint32_t family, diff --git a/frame/base/bli_cpuid.h b/frame/base/bli_cpuid.h index f2e4d334b..70c861e2f 100644 --- a/frame/base/bli_cpuid.h +++ b/frame/base/bli_cpuid.h @@ -5,6 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin + Copyright (C) 2018-2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -60,6 +61,7 @@ bool_t bli_cpuid_is_sandybridge( uint32_t family, uint32_t model, uint32_t fea bool_t bli_cpuid_is_penryn( uint32_t family, uint32_t model, uint32_t features ); // AMD +bool_t bli_cpuid_is_zen2( uint32_t family, uint32_t model, uint32_t features ); bool_t bli_cpuid_is_zen( uint32_t family, uint32_t model, uint32_t features ); bool_t bli_cpuid_is_excavator( uint32_t family, uint32_t model, uint32_t features ); bool_t bli_cpuid_is_steamroller( uint32_t family, uint32_t model, uint32_t features ); diff --git a/frame/base/bli_gks.c b/frame/base/bli_gks.c index 1f4f85b52..761bb18c5 100644 --- a/frame/base/bli_gks.c +++ b/frame/base/bli_gks.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018-2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -97,6 +97,11 @@ void bli_gks_init( void ) #endif // AMD architectures +#ifdef BLIS_CONFIG_ZEN2 + bli_gks_register_cntx( BLIS_ARCH_ZEN2, bli_cntx_init_zen2, + bli_cntx_init_zen2_ref, + bli_cntx_init_zen2_ind ); +#endif #ifdef BLIS_CONFIG_ZEN bli_gks_register_cntx( BLIS_ARCH_ZEN, bli_cntx_init_zen, bli_cntx_init_zen_ref, diff --git a/frame/include/bli_arch_config.h b/frame/include/bli_arch_config.h index 242700443..fee1f2a19 100644 --- a/frame/include/bli_arch_config.h +++ b/frame/include/bli_arch_config.h @@ -6,6 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP + Copyright (C) 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -61,7 +62,9 @@ CNTX_INIT_PROTS( penryn ) #endif // -- AMD64 architectures -- - +#ifdef BLIS_CONFIG_ZEN2 +CNTX_INIT_PROTS( zen2 ) +#endif #ifdef BLIS_CONFIG_ZEN CNTX_INIT_PROTS( zen ) #endif @@ -150,6 +153,9 @@ CNTX_INIT_PROTS( generic ) // -- AMD64 architectures -- +#ifdef BLIS_FAMILY_ZEN2 +#include "bli_family_zen2.h" +#endif #ifdef BLIS_FAMILY_ZEN #include "bli_family_zen.h" #endif diff --git a/frame/include/bli_type_defs.h b/frame/include/bli_type_defs.h index 8d60b1e62..4123472ef 100644 --- a/frame/include/bli_type_defs.h +++ b/frame/include/bli_type_defs.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018-2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -893,6 +893,7 @@ typedef enum BLIS_ARCH_PENRYN, // AMD + BLIS_ARCH_ZEN2, BLIS_ARCH_ZEN, BLIS_ARCH_EXCAVATOR, BLIS_ARCH_STEAMROLLER, @@ -915,7 +916,7 @@ typedef enum } arch_t; -#define BLIS_NUM_ARCHS 19 +#define BLIS_NUM_ARCHS 20 // From a042db011df9a1c3e7c7ac546541f4746b176ea5 Mon Sep 17 00:00:00 2001 From: Kiran Varaganti Date: Mon, 20 May 2019 14:17:32 +0530 Subject: [PATCH 37/53] Modified make_defs.mk for zen2 to get compiled by gcc version less than gcc9.0 Change-Id: I8fcac30538ee39534c296932639053b47b9a2d43 --- config/zen2/make_defs.mk | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/zen2/make_defs.mk b/config/zen2/make_defs.mk index dae064432..832681ca6 100644 --- a/config/zen2/make_defs.mk +++ b/config/zen2/make_defs.mk @@ -65,13 +65,15 @@ endif CKOPTFLAGS := $(COPTFLAGS) ifeq ($(CC_VENDOR),gcc) # gcc 9.0 (clang ?) or later: -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver2 +#CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver2 +# gcc 6.0 (clang 4.0) or later: +CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-avx256-split-unaligned-store # gcc 4.9 (clang 3.5) or later: # possibly add zen-specific instructions: -mclzero -madx -mrdseed -mmwaitx -msha -mxsavec -mxsaves -mclflushopt -mpopcnt #CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp else ifeq ($(CC_VENDOR),clang) -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver2 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-fma4 -mno-tbm -mno-xop -mno-lwp else $(error gcc or clang are required for this configuration.) endif From b80bd5bcb2be8551a9a21fafc8e6c8b6336c99b5 Mon Sep 17 00:00:00 2001 From: Kiran Varaganti Date: Tue, 21 May 2019 15:11:47 +0530 Subject: [PATCH 38/53] config/zen/bli_cntx_init_zen.c: removed BLIS_ENBLE_ZEN_BLOCK_SIZES macro. We have different configurations for both zen and zen2 config/zen/bli_family_zen.h: deleted macro BLIS_ENBLE_ZEN_BLOCK_SIZES config/zen/make_defs.mk: removed compiler flag -mno-avx256-split-unaligned-store frame/base/bli_cpuid.c: ROME family is 17H but model # is from 0x30H. test/test_gemm.c - commented out #define FILE_IN_OUT (some compilation error when BLIS is configured as amd64) Now we can use single configuration has ./configure amd64 - this will work both for ROME & Naples Change-Id: I91b4fc35380f8a35b4f4c345da040c6b5910b4a2 --- config/zen/bli_cntx_init_zen.c | 14 +++++--------- config/zen/bli_family_zen.h | 2 +- config/zen/make_defs.mk | 2 +- config/zen2/bli_family_zen2.h | 5 +++-- frame/base/bli_cpuid.c | 12 +++++++++++- test/test_gemm.c | 2 +- 6 files changed, 22 insertions(+), 15 deletions(-) diff --git a/config/zen/bli_cntx_init_zen.c b/config/zen/bli_cntx_init_zen.c index 84757ecae..049be6837 100644 --- a/config/zen/bli_cntx_init_zen.c +++ b/config/zen/bli_cntx_init_zen.c @@ -128,26 +128,22 @@ void bli_cntx_init_zen( cntx_t* cntx ) */ -#ifdef BLIS_ENABLE_ZEN_BLOCK_SIZES - // Zen optmized level 3 cache block sizes - #if BLIS_ENABLE_SINGLE_INSTANCE_BLOCK_SIZES + // Zen optmized level 3 cache block sizes + +#if BLIS_ENABLE_SINGLE_INSTANCE_BLOCK_SIZES bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 510, 144, 72 ); bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 1024, 256, 256 ); bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); - #else +#else bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 240, 144, 72 ); bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); - #endif -#else - bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); - bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); - bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); #endif + bli_blksz_init_easy( &blkszs[ BLIS_AF ], 8, 8, -1, -1 ); bli_blksz_init_easy( &blkszs[ BLIS_DF ], 8, 8, -1, -1 ); diff --git a/config/zen/bli_family_zen.h b/config/zen/bli_family_zen.h index 12242727e..5e1dc9bb2 100644 --- a/config/zen/bli_family_zen.h +++ b/config/zen/bli_family_zen.h @@ -42,7 +42,7 @@ #define BLIS_THREAD_MAX_IR 1 #define BLIS_THREAD_MAX_JR 1 -//#define BLIS_ENABLE_ZEN_BLOCK_SIZES + #define BLIS_ENABLE_SMALL_MATRIX #define BLIS_ENABLE_SMALL_MATRIX_TRSM diff --git a/config/zen/make_defs.mk b/config/zen/make_defs.mk index 5ef257107..a485af27b 100644 --- a/config/zen/make_defs.mk +++ b/config/zen/make_defs.mk @@ -64,7 +64,7 @@ endif CKOPTFLAGS := $(COPTFLAGS) ifeq ($(CC_VENDOR),gcc) # gcc 6.0 (clang 4.0) or later: -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-avx256-split-unaligned-store +CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 # gcc 4.9 (clang 3.5) or later: # possibly add zen-specific instructions: -mclzero -madx -mrdseed -mmwaitx -msha -mxsavec -mxsaves -mclflushopt -mpopcnt #CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp diff --git a/config/zen2/bli_family_zen2.h b/config/zen2/bli_family_zen2.h index a41c0222d..5160105b5 100644 --- a/config/zen2/bli_family_zen2.h +++ b/config/zen2/bli_family_zen2.h @@ -33,8 +33,8 @@ */ -//#ifndef BLIS_FAMILY_H -//#define BLIS_FAMILY_H +#ifndef BLI_FAMILY_ZEN2_ +#define BLI_FAMILY_ZEN2_ // By default, it is effective to parallelize the outer loops. // Setting these macros to 1 will force JR and IR inner loops @@ -57,4 +57,5 @@ #define BLIS_SMALL_MATRIX_A_THRES_M_SYRK 96 #define BLIS_SMALL_MATRIX_A_THRES_N_SYRK 128 +#endif diff --git a/frame/base/bli_cpuid.c b/frame/base/bli_cpuid.c index e4feb7249..aacca1da8 100644 --- a/frame/base/bli_cpuid.c +++ b/frame/base/bli_cpuid.c @@ -66,6 +66,14 @@ arch_t bli_cpuid_query_id( void ) // vendor. vendor = bli_cpuid_query( &family, &model, &features ); +#if 0 + printf( "vendor = %s\n", vendor==1 ? "AMD": "INTEL" ); + printf("family = %x\n", family ); + printf( "model = %x\n", model ); + + printf( "features = %x\n", features ); +#endif + if ( vendor == VENDOR_INTEL ) { // Check for each Intel configuration that is enabled, check for that @@ -252,7 +260,7 @@ bool_t bli_cpuid_is_zen2 // - 0x00-0xff (THIS NEEDS UPDATING) const bool_t is_arch = - ( 0x00 <= model && model <= 0xff ); + ( 0x30 <= model && model <= 0xff ); if ( !is_arch ) return FALSE; @@ -416,6 +424,8 @@ arch_t bli_cpuid_query_id( void ) //printf( "part = 0x%x\n", part ); //printf( "features = %u\n", features ); + + if ( vendor == VENDOR_ARM ) { if ( model == MODEL_ARMV8 ) diff --git a/test/test_gemm.c b/test/test_gemm.c index 823c95e0e..2d650260e 100644 --- a/test/test_gemm.c +++ b/test/test_gemm.c @@ -36,7 +36,7 @@ #include "blis.h" -#define FILE_IN_OUT // File based input matrix dimensions +//#define FILE_IN_OUT // File based input matrix dimensions //#define PRINT From ea082f839071dd9ec555062dc3851c31d12f00e4 Mon Sep 17 00:00:00 2001 From: kdevraje Date: Thu, 23 May 2019 10:38:29 +0530 Subject: [PATCH 39/53] adding empty zen2 directory with .gitignore file Change-Id: Ifa37cf54b2578aa19ad335372b44bca17043fe4b --- kernels/zen2/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 kernels/zen2/.gitignore diff --git a/kernels/zen2/.gitignore b/kernels/zen2/.gitignore new file mode 100644 index 000000000..5e7d2734c --- /dev/null +++ b/kernels/zen2/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore From 84215022f29fb3bfedd254d041635308d177e6c0 Mon Sep 17 00:00:00 2001 From: kdevraje Date: Thu, 23 May 2019 11:08:41 +0530 Subject: [PATCH 40/53] Adding threshold condition to dgemm small matrix kernels, defining the constants in zen2 configuration Change-Id: I53a58b5d734925a6fcb8d8bea5a02ddb8971fcd5 --- config/zen2/bli_family_zen2.h | 4 ++++ frame/3/gemm/bli_gemm_front.c | 7 +++++++ kernels/zen/3/bli_gemm_small.c | 4 ++++ 3 files changed, 15 insertions(+) diff --git a/config/zen2/bli_family_zen2.h b/config/zen2/bli_family_zen2.h index 5160105b5..e2a622b31 100644 --- a/config/zen2/bli_family_zen2.h +++ b/config/zen2/bli_family_zen2.h @@ -57,5 +57,9 @@ #define BLIS_SMALL_MATRIX_A_THRES_M_SYRK 96 #define BLIS_SMALL_MATRIX_A_THRES_N_SYRK 128 +#define BLIS_ENABLE_SMALL_MATRIX_ROME +#define BLIS_SMALL_MATRIX_THRES_ROME 400 + + #endif diff --git a/frame/3/gemm/bli_gemm_front.c b/frame/3/gemm/bli_gemm_front.c index aaabed200..cc61dfdab 100644 --- a/frame/3/gemm/bli_gemm_front.c +++ b/frame/3/gemm/bli_gemm_front.c @@ -53,6 +53,13 @@ void bli_gemm_front obj_t b_local; obj_t c_local; + gint_t M = bli_obj_length( c ); + gint_t N = bli_obj_width( c ); + gint_t K = bli_obj_width( a ); + + if( !(M && N && K)) return; + + #ifdef BLIS_ENABLE_SMALL_MATRIX // Only handle small problems separately for homogeneous datatypes. if ( bli_obj_dt( a ) == bli_obj_dt( b ) && diff --git a/kernels/zen/3/bli_gemm_small.c b/kernels/zen/3/bli_gemm_small.c index 510dab010..d500bf55d 100644 --- a/kernels/zen/3/bli_gemm_small.c +++ b/kernels/zen/3/bli_gemm_small.c @@ -1582,8 +1582,12 @@ static err_t bli_dgemm_small // If alpha is zero, scale by beta and return. // printf("alpha_cast = %f beta_cast = %f [ Trans = %d %d], [stride = %d %d %d] [m,n,k = %d %d %d]\n",*alpha_cast,*beta_cast, bli_obj_has_trans( a ), bli_obj_has_trans( b ), lda, ldb,ldc, M,N,K); +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if( N < BLIS_SMALL_MATRIX_THRES_ROME && K < BLIS_SMALL_MATRIX_THRES_ROME) +#else if ((((L) < (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES)) || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) +#endif { guint_t lda = bli_obj_col_stride( a ); // column stride of matrix OP(A), where OP(A) is Transpose(A) if transA enabled. From 02920f5c480c42706b487e37b5ecc96c3555b851 Mon Sep 17 00:00:00 2001 From: kdevraje Date: Thu, 23 May 2019 15:29:59 +0530 Subject: [PATCH 41/53] make checkblis fails for matrix dimension check at the begining hence reverting it Change-Id: Ibd2ee8c2d4914598b72003fbfc5845be9c9c1e87 --- frame/3/gemm/bli_gemm_front.c | 4 ++-- kernels/zen/3/bli_gemm_small.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frame/3/gemm/bli_gemm_front.c b/frame/3/gemm/bli_gemm_front.c index cc61dfdab..0d83104e1 100644 --- a/frame/3/gemm/bli_gemm_front.c +++ b/frame/3/gemm/bli_gemm_front.c @@ -52,13 +52,13 @@ void bli_gemm_front obj_t a_local; obj_t b_local; obj_t c_local; - +#if 0 gint_t M = bli_obj_length( c ); gint_t N = bli_obj_width( c ); gint_t K = bli_obj_width( a ); if( !(M && N && K)) return; - +#endif #ifdef BLIS_ENABLE_SMALL_MATRIX // Only handle small problems separately for homogeneous datatypes. diff --git a/kernels/zen/3/bli_gemm_small.c b/kernels/zen/3/bli_gemm_small.c index d500bf55d..dd2c1ce58 100644 --- a/kernels/zen/3/bli_gemm_small.c +++ b/kernels/zen/3/bli_gemm_small.c @@ -1583,7 +1583,7 @@ static err_t bli_dgemm_small // If alpha is zero, scale by beta and return. // printf("alpha_cast = %f beta_cast = %f [ Trans = %d %d], [stride = %d %d %d] [m,n,k = %d %d %d]\n",*alpha_cast,*beta_cast, bli_obj_has_trans( a ), bli_obj_has_trans( b ), lda, ldb,ldc, M,N,K); #ifdef BLIS_ENABLE_SMALL_MATRIX_ROME - if( N < BLIS_SMALL_MATRIX_THRES_ROME && K < BLIS_SMALL_MATRIX_THRES_ROME) + if( (L != 0) && (K != 0) && (N < BLIS_SMALL_MATRIX_THRES_ROME) && (K < BLIS_SMALL_MATRIX_THRES_ROME)) #else if ((((L) < (D_BLIS_SMALL_MATRIX_THRES * D_BLIS_SMALL_MATRIX_THRES)) || ((M < D_BLIS_SMALL_M_RECT_MATRIX_THRES) && (K < D_BLIS_SMALL_K_RECT_MATRIX_THRES))) && ((L!=0) && (K!=0))) From e05171118c377f356f89c4daf8a0d5ddc5a4e4f7 Mon Sep 17 00:00:00 2001 From: Meghana Date: Thu, 23 May 2019 16:15:27 +0530 Subject: [PATCH 42/53] Implemented TRSM for small matrices for cases where A is on the right Added separate kernels for zen and zen2 Change-Id: I6318ddc250cf82516c1aa4732718a35eae0c9134 --- kernels/zen/3/bli_trsm_small.c | 14736 +++++++++++++++--- kernels/zen2/3/bli_trsm_small.c | 25023 ++++++++++++++++++++++++++++++ 2 files changed, 37393 insertions(+), 2366 deletions(-) create mode 100644 kernels/zen2/3/bli_trsm_small.c diff --git a/kernels/zen/3/bli_trsm_small.c b/kernels/zen/3/bli_trsm_small.c index ca8e5b142..21114829d 100644 --- a/kernels/zen/3/bli_trsm_small.c +++ b/kernels/zen/3/bli_trsm_small.c @@ -34,18 +34,122 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "blis.h" #ifdef BLIS_ENABLE_SMALL_MATRIX_TRSM +#ifndef BLI_FAMILY_ZEN2_ #include "immintrin.h" - #define GEMM_BLK_V1 8 //Block size to perform gemm and apply trsm #define GEMM_ACCUM_A 1 //Peform B1=B1-(B0*A0) operation instead of B1'=(B0*A0) and then B1=B1-B1' #define OPT_CACHE_BLOCKING_L1 1 //Perform trsm block-wise in blocks of GEMM_BLK_V1 instead of all columns of B together. #define REARRANGE_SHFL 0 //Rearrange operations using blend or shuffle #define BLI_AlXB_M_SP 16 -#define BLI_AlXB_M_DP 16 +//#define BLI_AlXB_M_DP 16 #define BLI_XAltB_N_SP 128 -#define BLI_XAltB_N_DP 64 #define BLI_AutXB_M_SP 64 #define BLI_AutXB_N_SP 128 +#define max(a,b) a>b?a:b +#define min(a,b) abuffer; - double *B = b->buffer; - - if (m != BLI_AlXB_M_DP || (n&3) != 0) + for (k = 0; k < M; k++) { - return BLIS_NOT_YET_IMPLEMENTED; + double lkk_inv = 1.0/A[k+k*lda]; + for (j = 0; j < N; j++) + { + B[k + j*ldb] *= lkk_inv; + for (i = k+1; i < M; i++) + { + B[i + j*ldb] -= A[i + k*lda] * B[k + j*ldb]; + } + } + }// k -loop + return BLIS_SUCCESS; +}// end of function + +/* TRSM scalar code for the case AX = alpha * B + * A is lower-triangular, unit-diagonal, no transpose + * Dimensions: A: mxm X: mxn B:mxn + */ + +static err_t dtrsm_small_AlXB_unitDiag ( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb + ) +{ + + dim_t i, j, k; + + for (k = 0; k < M; k++) + { + for (j = 0; j < N; j++) + { + for (i = k+1; i < M; i++) + { + B[i + j*ldb] -= A[i + k*lda] * B[k + j*ldb]; + } + } + } + return BLIS_SUCCESS; +}// end of function + +/* TRSM scalar code for the case XA = alpha * B + * A is upper-triangular, non-unit-diagonal no transpose + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAuB ( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + for(k = 0; k < N; k++) + { + double lkk_inv = 1.0/A[k+k*lda]; + for(i = 0; i < M; i++) + { + B[i+k*ldb] *= lkk_inv; + for(j = k+1; j < N; j++) + { + B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; + } + } + + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is lower-triangular, non-unit triangular, no transpose + * Dimensions: X:mxn A:nxn B:mxn + */ + +static err_t dtrsm_small_XAlB ( + double *A, + double *B, + double alpha, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + for(i = 0; i < M; i++) + for(j = 0; j < N; j++) + B[i+j*ldb] *= alpha; + + for(k = N-1; k+1 > 0; k--) + { + double lkk_inv = 1.0/A[k+k*lda]; + for(i = M-1; i+1 > 0; i--) + { + B[i+k*ldb] *= lkk_inv; + for(j = k-1; j+1 > 0; j--) + { + B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is lower-triangular, unit-diagonal, no transpose + *Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAlB_unitDiag( + double *A, + double *B, + double alpha, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(i = 0 ; i < M; i++) + for(j = 0; j < N; j++) + B[i+j*ldb] *= alpha; + + for(k = N-1; k+1 > 0; k--) + { + for(i = M-1; i+1 > 0; i--) + { + for(j = k-1; j+1 > 0; j--) + { + B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + *A is upper-triangular, non-unit-diagonal, A is transposed + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAutB ( + double *A, + double *B, + double alpha, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(i = 0; i < M; i++) + for(j = 0; j < N; j++) + B[i+j*ldb] *=alpha; + + for(k = N-1; k+1 > 0; k--) + { + double lkk_inv = 1.0/A[k+k*lda]; + for(i = M-1; i+1 > 0; i--) + { + B[i+k*ldb] *= lkk_inv; + for(j = k-1; j+1 > 0; j--) + { + B[i+j*ldb] -= B[i+k*ldb] * A[j+k*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is upper-triangular, unit-diagonal, A has to be transposed + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAutB_unitDiag( + double *A, + double *B, + double alpha, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(i = 0; i< M; i++) + for(j = 0; j< N; j++) + B[i+j*ldb] *= alpha; + + for(i = M-1; i+1 > 0; i--) + { + for(j = N-1; j+1 > 0; j--) + { + for(k = j-1; k+1 > 0; k--) + { + B[i+k*ldb] -= B[i+j*ldb] * A[k+j*lda]; + + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is lower-triangular, non-unit-diagonal, A has to be transposed + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAltB ( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(k = 0; k < N; k++) + { + double lkk_inv = 1.0/A[k+k*lda]; + for(i = 0; i < M; i++) + { + B[i+k*ldb] *= lkk_inv; + for(j = k+1; j < N; j++) + { + B[i+j*ldb] -= B[i+k*ldb] * A[j+k*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for XA = alpha * B + * A is lower-triangular, unit-diagonal, A has to be transposed + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAltB_unitDiag( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(k = 0; k < N; k++) + { + for(i = 0; i < M; i++) + { + for(j = k+1; j < N; j++) + { + B[i+j*ldb] -= B[i+k*ldb] * A[j+k*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is upper-triangular, unit-diagonal, no transpose + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAuB_unitDiag ( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(k = 0; k < N; k++) + { + for(i = 0; i < M; i++) + { + for(j = k+1; j < N; j++) + { + B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM for the case AX = alpha * B, Double precision + * A is lower-triangular, no-transpose, non-unit diagonal + * dimensions A: mxm X: mxn B: mxn + + b01---> + * ***************** + ** * * * * * + * * * * * * * + * * *b01* * * * + * * * * * * * +a10 ****** b11 ***************** + | * * * | * * * * * + | * * * | * * * * * + | *a10*a11* | *b11* * * * + v * * * v * * * * * + *********** ***************** + * * * * * * * * * + * * * * * * * * * + * * * * * * * * * + * * * * * * * * * + **************** ***************** + a11---> +*/ + +static err_t bli_dtrsm_small_AlXB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + dim_t D_MR = 4; //size of block along 'M' dimpension + dim_t D_NR = 8; //size of block along 'N' dimension + + dim_t m = bli_obj_length(b); // number of rows of matrix B + dim_t n = bli_obj_width(b); // number of columns of matrix B + + if(max(m,n) > 90) + return BLIS_NOT_YET_IMPLEMENTED; + + dim_t m_remainder = m % D_MR; //number of remainder rows + dim_t n_remainder = n % D_NR; //number of remainder columns + + dim_t cs_a = bli_obj_col_stride(a); // column stride of A + dim_t cs_b = bli_obj_col_stride(b); // column stride of B + + dim_t i, j, k; //loop variables + dim_t k_iter; //number of times GEMM to be performed + + double AlphaVal = *(double *)AlphaObj->buffer; //value of alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a10, *a11, *b01, *b11; //pointers that point to blocks for GEMM and TRSM + double *ptr_b01_dup; + + double ones = 1.0; + + //scratch registers + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + + + for(j = 0; j+D_NR-1 < n; j += D_NR) //loop along 'N' dimension + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' dimension + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM to be performed(in blocks of 4x4) + + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + ///GEMM code begins/// + + for(k = 0; k< k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm16 = _mm256_loadu_pd((double const *)(a10));//A10[0][0] A10[1][0] A10[2][0] A10[3][0] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a));//A10[0][1] A10[1][1] A10[2][1] A10[3][1] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2));//A10[0][2] A10[1][2] A10[2][2] A10[3][2] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][4]*A10[3][2]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][5]*A10[3][2]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][6]*A10[3][2]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][7]*A10[3][2]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3));//A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[3][0] B01[3][0]*A10[2][3] B01[3][0]*A10[3][0]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[3][1]*A10[0][3] B01[3][1]*A10[3][0] B01[3][1]*A10[2][3] B01[3][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[3][2]*A10[0][3] B01[3][2]*A10[3][0] B01[3][2]*A10[2][3] B01[3][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[3][3]*A10[0][3] B01[3][3]*A10[3][0] B01[3][3]*A10[2][3] B01[3][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[3][4]*A10[0][3] B01[3][4]*A10[3][0] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[3][5]*A10[0][3] B01[3][5]*A10[3][0] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[3][6]*A10[0][3] B01[3][6]*A10[3][0] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[3][7]*A10[0][3] B01[3][7]*A10[3][0] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to calculate next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to calculate next block of B for GEMM + } + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha + + ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] * alpha -= B01[0-3][0] + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] * alpha -= B01[0-3][1] + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] * alpha -= B01[0-3][2] + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] * alpha -= B01[0-3][3] + ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] * alpha -= B01[0-3][4] + ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] * alpha -= B01[0-3][5] + ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] * alpha -= B01[0-3][6] + ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] * alpha -= B01[0-3][7] + + ///implement TRSM/// + + ///transpose of B11// + ///unpacklow/// + ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[2][4] B11[2][5] + ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[2][6] B11[2][7] + + //rearrange low elements + ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] + ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[1][4] B11[1][5] B11[3][4] B11[3][5] + ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[1][6] B11[1][7] B11[3][6] B11[3][7] + + //rearrange high elements + ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] + ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] + + ymm0 = _mm256_broadcast_sd((double const *)&ones); + + //broadcast diagonal elements of A11 + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] + ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] + + ymm5 = _mm256_unpacklo_pd(ymm1, ymm2); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] + ymm6 = _mm256_unpacklo_pd(ymm3, ymm4); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] + + ymm5 = _mm256_blend_pd(ymm5, ymm6, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm0 = _mm256_div_pd(ymm0, ymm5); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2] + + //extract a00 + ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + + //(Row 0): perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm1); //B11[0-3][0] /= A11[0][0] + ymm12 = _mm256_mul_pd(ymm12, ymm1); //B11[0-3][4] /= A11[0][0] + + //extract a11 + ymm1 = _mm256_permute_pd(ymm0, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] + + a11 += cs_a; + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= A11[1][0] * B11[0-3][0] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= A11[2][0] * B11[0-3][0] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= A11[3][0] * B11[0-3][0] + + ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= A11[1][0] * B11[0-3][4] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= A11[2][0] * B11[0-3][4] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= A11[3][0] * B11[0-3][4] + + ymm9 = _mm256_mul_pd(ymm9, ymm1); //B11[0-3][1] /= A11[1][1] + ymm13 = _mm256_mul_pd(ymm13, ymm1); //B11[0-3][5] /= A11[1][1] + + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] + + a11 += cs_a; + + //extract a22 + ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A110[][0] 1/A11[2][2] 1/A11[2][2] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(ROw2): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] + + ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] + + //perform mul operation + ymm10 = _mm256_mul_pd(ymm10, ymm1); //B11[0-3][2] /= A11[2][2] + ymm14 = _mm256_mul_pd(ymm14, ymm1); //B11[0-3][6] /= A11[2][2] + + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] + + a11 += cs_a; + + //extract a33 + ymm1 = _mm256_permute_pd(ymm0, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11);//1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(ROw2): FMA operations + ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[3][0-3] -= A11[3][2] * B11[0-3][2] + + ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[7][0-3] -= A11[3][2] * B11[0-3][6] + + //perform mul operation + ymm11 = _mm256_mul_pd(ymm11, ymm1); //B11[0-3][3] /= A11[3][3] + ymm15 = _mm256_mul_pd(ymm15, ymm1); //B11[0-3][7] /= A11[3][3] + + //unpacklow// + ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] + ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + + ///unpack high/// + ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[4][1] B11[5][1] B11[4][3] B11[5][3] + ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[6][1] B11[7][1] B11[6][3] B11[7][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store B11[0][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store B11[1][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store B11[2][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store B11[3][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store B11[4][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store B11[5][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store B11[6][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store B11[7][0-3] + } + + if(m_remainder) //implementation for reamainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM operation to be done(in blocks of 4x4) + + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + ///GEMM code Begins/// + for(k = 0; k< k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm16 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0] ) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm16 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 1)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] + + b01 += 1; //move to next row of B01 + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] //A10[1][2] A10[2][2] A10[3][2] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][0]*A10[3][2]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][1]*A10[3][2]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][2]*A10[3][2]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm8 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm8 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm8 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm8 += (B01[3][0]*A10[0][3] B01[3][4]*A10[1][3] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm8 += (B01[3][1]*A10[0][3] B01[3][5]*A10[1][3] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm8 += (B01[3][2]*A10[0][3] B01[3][6]*A10[1][3] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm8 += (B01[3][3]*A10[0][3] B01[3][7]*A10[1][3] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + } + + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value + + ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] *alpha -= B01[0-3][0] + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] *alpha -= B01[0-3][1] + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] *alpha -= B01[0-3][2] + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] *alpha -= B01[0-3][3] + ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] *alpha -= B01[0-3][4] + ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] *alpha -= B01[0-3][5] + ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] *alpha -= B01[0-3][6] + ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] *alpha -= B01[0-3][7] + + ///implement TRSM/// + + ///unpacklow/// + ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[1][4] B11[1][5] + ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[1][6] B11[1][7] + + //rearrange low elements + ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] + ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[5][0] B11[5][1] B11[7][0] B11[7][1] + ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[5][2] B11[5][3] B11[7][2] B11[7][3] + + //rearrange high elements + ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] + ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] + + + ymm0 = _mm256_broadcast_sd((double const *)&ones); + + //broadcast diagonal elements of A11 + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] + ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] + + ymm5 = _mm256_unpacklo_pd(ymm1, ymm2); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] + ymm6 = _mm256_unpacklo_pd(ymm3, ymm4); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] + + ymm5 = _mm256_blend_pd(ymm5, ymm6, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm0 = _mm256_div_pd(ymm0, ymm5); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a00 + ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + + //(Row 0): perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm1); //B11[0-3][0] /= A11[0][0] + ymm12 = _mm256_mul_pd(ymm12, ymm1); //B11[0-3][4] /= A11[0][0] + + //extract a11 + ymm1 = _mm256_permute_pd(ymm0, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] + + a11 += cs_a; + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= B11[0-3][0]*A11[1][0] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= B11[0-3][0]*A11[2][0] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= B11[0-3][0]*A11[3][0] + + ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= B11[0-3][4]*A11[1][4] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= B11[0-3][4]*A11[2][4] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= B11[0-3][4]*A11[3][4] + + ymm9 = _mm256_mul_pd(ymm9, ymm1); //B11[0-3][1] /= A11[1][1] + ymm13 = _mm256_mul_pd(ymm13, ymm1); //B11[0-3][5] /= A11[1][1] + + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] + + a11 += cs_a; + + //extract a22 + ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(ROw2): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] + + ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] + + //perform mul operation + ymm10 = _mm256_mul_pd(ymm10, ymm1); //B11[0-3][2] /=A11[2][2] + ymm14 = _mm256_mul_pd(ymm14, ymm1); //B11[0-3][6] /= A11[2][2] + + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] + + a11 += cs_a; + + //extract a33 + ymm1 = _mm256_permute_pd(ymm0, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(ROw2): FMA operations + ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[0-3][3] -= A11[3][2]*B11[0-3][2] + + ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[0-3][7] -= A11[3][2]*B11[0-3][6] + + //perform mul operation + ymm11 = _mm256_mul_pd(ymm11, ymm1); //B11[0-3][3] /= A11[3][3] + ymm15 = _mm256_mul_pd(ymm15, ymm1); //B11[0-3][7] /= A11[3][3] + + //unpacklow// + ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] + ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + + ///unpack high/// + ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[0][5] B11[1][5] B11[0][7] B11[1][7] + ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[2][5] B11[3][5] B11[2][7] B11[3][7] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm8 = _mm256_loadu_pd((double const *)(b11 + cs_b * 0)); //load B11[0-3][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b * 1)); //load B11[0-3][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //load B11[0-3][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //load B11[0-3][3] + ymm12 = _mm256_loadu_pd((double const *)(b11 + cs_b * 4)); //load B11[0-3][4] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b * 5)); //load B11[0-3][5] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b * 6)); //load B11[0-3][6] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b * 7)); //load B11[0-3][7] + //determine correct values to store + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x08); + ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x08); + ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x08); + ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x08); + ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x08); + ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x08); + ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x08); + ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x08); + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm0, ymm8, 0x30); + ymm1 = _mm256_permute2f128_pd(ymm1, ymm9, 0x30); + ymm2 = _mm256_permute2f128_pd(ymm2, ymm10, 0x30); + ymm3 = _mm256_permute2f128_pd(ymm3, ymm11, 0x30); + ymm4 = _mm256_permute2f128_pd(ymm4, ymm12, 0x30); + ymm5 = _mm256_permute2f128_pd(ymm5, ymm13, 0x30); + ymm6 = _mm256_permute2f128_pd(ymm6, ymm14, 0x30); + ymm7 = _mm256_permute2f128_pd(ymm7, ymm15, 0x30); + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x0E); + ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x0E); + ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x0E); + ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x0E); + ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x0E); + ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x0E); + ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x0E); + ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x0E); + } + + _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store(B11[0-3][4]) + _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store(B11[0-3][5]) + _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store(B11[0-3][6]) + _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store(B11[0-3][7]) + + } } - alphaVal = *((double *)AlphaObj->buffer); - - /* Small _GEMM preparation code */ - bli_obj_create( BLIS_DOUBLE, 1, 1, 0, 0, &alpha ); - bli_obj_create( BLIS_DOUBLE, 1, 1, 0, 0, &beta ); - - /* B = B - A*B */ - bli_setsc( -(1.0), 0.0, &alpha ); - bli_setsc( (1.0), 0.0, &beta ); - - bli_obj_create_with_attached_buffer( BLIS_DOUBLE, blk_size, blk_size, a->buffer, rsa, lda, &Ga); - bli_obj_create_with_attached_buffer( BLIS_DOUBLE, blk_size, n, b->buffer, rsb, ldb, &Gb); - bli_obj_create_with_attached_buffer( BLIS_DOUBLE, blk_size, n, b->buffer, rsb, ldb, &Gc); - - bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Ga ); - bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gb ); - bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gc ); - - //first block of trsm - Gb.buffer = (void*)(B + i); - - if (alphaVal != 1) + if((n & 4)) //implementation for remainder columns(when 'N' is a multiple of 4) { - if (isUnitDiag == 0) - { - blis_dtrsm_microkernel_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - fp_blis_dtrsm_microkernel = blis_dtrsm_microkernel; - } - else - { - blis_dtrsm_microkernel_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - fp_blis_dtrsm_microkernel = blis_dtrsm_microkernel_unitDiag; - } - bli_setsc( alphaVal, 0.0, &beta ); - } - else - { - if (isUnitDiag == 0) - { - blis_dtrsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - fp_blis_dtrsm_microkernel = blis_dtrsm_microkernel; - } - else - { - blis_dtrsm_microkernel_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - fp_blis_dtrsm_microkernel = blis_dtrsm_microkernel_unitDiag; - } + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM to be performed(in block of 4) + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b*3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a*2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[1][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[2][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[3][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B01[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B01[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B01[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + //compute reciprocals of L(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[0][0] A11[2][2] A11[2][2] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[1][1] A11[1][1] A11[3][3] A11[3][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + ymm4 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm4); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] +/* + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); +*/ + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] +/* + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); +*/ + //extract a00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm4 = _mm256_mul_pd(ymm4, ymm15); //B11[0][0-3] /= A11[0][0] + + //extract diag a11 from a + ymm15 = _mm256_permute_pd(ymm14, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]*B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]*B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]*B11[0][0-3] + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm15); //B11[1][0-3] /= A11[1][1] + + + //extract diag a22 from a + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]*B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]*B11[1][0-3] + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + ymm11 = _mm256_mul_pd(ymm11, ymm15); //B11[2][0-3] /= A11[2][2] + + //extract diag a33 from a + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]*B11[2][0-3] + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + ymm13 = _mm256_mul_pd(ymm13, ymm15); //B11[3][0-3] /= A11[3][3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b*2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[0-3][3]) + + } + if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + + k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) //looop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + //compute reciprocals of L(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + ymm4 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm4); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] +/* + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); +*/ + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] +/* + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); +*/ + //extract a00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00);//1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm4 = _mm256_mul_pd(ymm4, ymm15); //B11[0][0-3] /= A11[0][0] + + //extract diag a11 from a + ymm15 = _mm256_permute_pd(ymm14, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[][] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]* B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]* B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]* B11[0][0-3] + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm15); //B11[1][0-3] /= A11[1][1] + + + //extract diag a22 from a + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]* B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]* B11[1][0-3] + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + ymm11 = _mm256_mul_pd(ymm11, ymm15); //B11[2][0-3] /= A11[2][2] + + //extract diag a33 from a + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]* B11[2][0-3] + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + ymm13 = _mm256_mul_pd(ymm13, ymm15); //B11[3][0-3] /= A11[3][3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //load 4x4 block from b11 + ymm4 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][2] B11[3][3] + + //determine correct values to store + + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x08); + ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x08); + ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x08); + ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x08); + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm0, ymm4,0x30); + ymm1 = _mm256_permute2f128_pd(ymm1, ymm5,0x30); + ymm2 = _mm256_permute2f128_pd(ymm2, ymm6,0x30); + ymm3 = _mm256_permute2f128_pd(ymm3, ymm7,0x30); + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x0E); + ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x0E); + ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x0E); + ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x0E); + } + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) + + } + + n_remainder -= 4; + j += 4; + } - -//gemm update - for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + if(n_remainder) //implementation fo remaining columns(when 'N' is not a multiple of D_NR) { - Ga.buffer = (void*)(L + j + i*lda); - Gc.buffer = (void*)(B + j); - bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha Value + + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + if(n_remainder == 3) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_broadcast_sd((double const *)&ones); + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const*)&ones); + } + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + } + + ///GEMM code ends/// + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + //compute reciprocals of L(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + ymm4 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm4); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] +/* + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); +*/ + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] +/* + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); +*/ + //extract a00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm4 = _mm256_mul_pd(ymm4, ymm15); //B11[0][0-3] /= A11[0][0] + + //extract diag a11 from a + ymm15 = _mm256_permute_pd(ymm14, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0] * B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0] * B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0] * B11[0][0-3] + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm15); //B11[1][0-3] /= A11[1][1] + + + //extract diag a22 from a + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1] * B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1] * B11[1][0-3] + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + ymm11 = _mm256_mul_pd(ymm11, ymm15); //B11[2][0-3] /= A11[2][2] + + //extract diag a33 from a + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2] * B11[2][0-3] + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + ymm13 = _mm256_mul_pd(ymm13, ymm15); //B11[3][0-3] /= A11[3][3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + } + + } + if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + + k_iter = i / D_MR; //number of times GEMM operations to be performed + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha value + + ///GEMM for previously calculated values /// + + + //load 4x4 block from b11 + if(n_remainder == 3) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_broadcast_sd((double const *)&ones); + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[0][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[1][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[2][3] + + b01 += 1; //move to next row of B + + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[3][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm8 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] * alpha -= ymm4 + ymm9 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] * alpha -= ymm5 + ymm10 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] * alpha -= ymm6 + ymm11 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] * alpha -= ymm7 + + ///implement TRSM/// + //determine correct values to store + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x08); + ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x08); + ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x08); + ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x08); + + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm8, ymm0, 0x30); + ymm1 = _mm256_permute2f128_pd(ymm9, ymm1, 0x30); + ymm2 = _mm256_permute2f128_pd(ymm10, ymm2, 0x30); + ymm3 = _mm256_permute2f128_pd(ymm11, ymm3, 0x30); + + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x0E); + ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x0E); + ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x0E); + ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x0E); + } + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + } + + ///scalar code for trsm without alpha/// + dtrsm_small_AlXB(a11, b11, m_remainder, n_remainder, cs_a, cs_b); + } } - bli_setsc( (1.0), 0.0, &beta ); - - //trsm of remaining blocks - for (i = blk_size; i < m; i += blk_size) - { - Gb.buffer = (void*)(B + i); - - fp_blis_dtrsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - - - for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT - { - Ga.buffer = (void*)(L + j + i*lda); - Gc.buffer = (void*)(B + j); - - bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb - } - - } // End of for loop - i - return BLIS_SUCCESS; +} +/* TRSM for the case AX = alpha * B, Double precision + * A is lower-triangular, no-transpose, unit diagonal + * dimensions A: mxm X: mxn B: mxn + + b01---> + * ***************** + ** * * * * * + * * * * * * * + * * *b01* * * * + * * * * * * * +a10 ****** b11 ***************** + | * * * | * * * * * + | * * * | * * * * * + | *a10*a11* | *b11* * * * + v * * * v * * * * * + *********** ***************** + * * * * * * * * * + * * * * * * * * * + * * * * * * * * * + * * * * * * * * * + **************** ***************** + a11---> +*/ + +static err_t bli_dtrsm_small_AlXB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + dim_t D_MR = 4; //size of block along 'M' dimpension + dim_t D_NR = 8; //size of block along 'N' dimension + + dim_t m = bli_obj_length(b); // number of rows of matrix B + dim_t n = bli_obj_width(b); // number of columns of matrix B + + if(max(m,n) > 90) + return BLIS_NOT_YET_IMPLEMENTED; + + dim_t m_remainder = m % D_MR; //number of remainder rows + dim_t n_remainder = n % D_NR; //number of remainder columns + + dim_t cs_a = bli_obj_col_stride(a); // column stride of A + dim_t cs_b = bli_obj_col_stride(b); // column stride of B + + dim_t i, j, k; //loop variables + dim_t k_iter; //number of times GEMM to be performed + + double AlphaVal = *(double *)AlphaObj->buffer; //value of alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a10, *a11, *b01, *b11; //pointers that point to blocks for GEMM and TRSM + double *ptr_b01_dup; + + double ones = 1.0; + + //scratch registers + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + + + for(j = 0; j+D_NR-1 < n; j += D_NR) //loop along 'N' dimension + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' dimension + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM to be performed(in blocks of 4x4) + + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + ///GEMM code begins/// + + for(k = 0; k< k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm16 = _mm256_loadu_pd((double const *)(a10));//A10[0][0] A10[1][0] A10[2][0] A10[3][0] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a));//A10[0][1] A10[1][1] A10[2][1] A10[3][1] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2));//A10[0][2] A10[1][2] A10[2][2] A10[3][2] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][4]*A10[3][2]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][5]*A10[3][2]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][6]*A10[3][2]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][7]*A10[3][2]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3));//A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[3][0] B01[3][0]*A10[2][3] B01[3][0]*A10[3][0]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[3][1]*A10[0][3] B01[3][1]*A10[3][0] B01[3][1]*A10[2][3] B01[3][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[3][2]*A10[0][3] B01[3][2]*A10[3][0] B01[3][2]*A10[2][3] B01[3][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[3][3]*A10[0][3] B01[3][3]*A10[3][0] B01[3][3]*A10[2][3] B01[3][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[3][4]*A10[0][3] B01[3][4]*A10[3][0] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[3][5]*A10[0][3] B01[3][5]*A10[3][0] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[3][6]*A10[0][3] B01[3][6]*A10[3][0] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[3][7]*A10[0][3] B01[3][7]*A10[3][0] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to calculate next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to calculate next block of B for GEMM + } + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha + + ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] * alpha -= B01[0-3][0] + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] * alpha -= B01[0-3][1] + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] * alpha -= B01[0-3][2] + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] * alpha -= B01[0-3][3] + ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] * alpha -= B01[0-3][4] + ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] * alpha -= B01[0-3][5] + ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] * alpha -= B01[0-3][6] + ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] * alpha -= B01[0-3][7] + + ///implement TRSM/// + + ///transpose of B11// + ///unpacklow/// + ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[2][4] B11[2][5] + ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[2][6] B11[2][7] + + //rearrange low elements + ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] + ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[1][4] B11[1][5] B11[3][4] B11[3][5] + ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[1][6] B11[1][7] B11[3][6] B11[3][7] + + //rearrange high elements + ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] + ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] + + //broadcast diagonal elements of A11 + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] + ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] + + a11 += cs_a; + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= A11[1][0] * B11[0-3][0] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= A11[2][0] * B11[0-3][0] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= A11[3][0] * B11[0-3][0] + + ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= A11[1][0] * B11[0-3][4] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= A11[2][0] * B11[0-3][4] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= A11[3][0] * B11[0-3][4] + + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] + + a11 += cs_a; + + //(ROw2): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] + + ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] + + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] + + a11 += cs_a; + + //(ROw2): FMA operations + ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[3][0-3] -= A11[3][2] * B11[0-3][2] + + ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[7][0-3] -= A11[3][2] * B11[0-3][6] + + //unpacklow// + ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] + ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + + ///unpack high/// + ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[4][1] B11[5][1] B11[4][3] B11[5][3] + ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[6][1] B11[7][1] B11[6][3] B11[7][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store B11[0][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store B11[1][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store B11[2][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store B11[3][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store B11[4][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store B11[5][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store B11[6][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store B11[7][0-3] + } + + if(m_remainder) //implementation for reamainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM operation to be done(in blocks of 4x4) + + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + ///GEMM code Begins/// + for(k = 0; k< k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm16 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0] ) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm16 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 1)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] + + b01 += 1; //move to next row of B01 + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] //A10[1][2] A10[2][2] A10[3][2] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][0]*A10[3][2]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][1]*A10[3][2]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][2]*A10[3][2]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm8 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm8 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm8 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm8 += (B01[3][0]*A10[0][3] B01[3][4]*A10[1][3] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm8 += (B01[3][1]*A10[0][3] B01[3][5]*A10[1][3] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm8 += (B01[3][2]*A10[0][3] B01[3][6]*A10[1][3] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm8 += (B01[3][3]*A10[0][3] B01[3][7]*A10[1][3] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + } + + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value + + ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] *alpha -= B01[0-3][0] + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] *alpha -= B01[0-3][1] + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] *alpha -= B01[0-3][2] + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] *alpha -= B01[0-3][3] + ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] *alpha -= B01[0-3][4] + ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] *alpha -= B01[0-3][5] + ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] *alpha -= B01[0-3][6] + ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] *alpha -= B01[0-3][7] + + ///implement TRSM/// + + ///unpacklow/// + ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[1][4] B11[1][5] + ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[1][6] B11[1][7] + + //rearrange low elements + ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] + ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[5][0] B11[5][1] B11[7][0] B11[7][1] + ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[5][2] B11[5][3] B11[7][2] B11[7][3] + + //rearrange high elements + ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] + ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] + + //broadcast diagonal elements of A11 + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] + ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] + + a11 += cs_a; + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= B11[0-3][0]*A11[1][0] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= B11[0-3][0]*A11[2][0] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= B11[0-3][0]*A11[3][0] + + ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= B11[0-3][4]*A11[1][4] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= B11[0-3][4]*A11[2][4] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= B11[0-3][4]*A11[3][4] + + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] + + a11 += cs_a; + + //(ROw2): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] + + ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] + + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] + + a11 += cs_a; + + //(ROw2): FMA operations + ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[0-3][3] -= A11[3][2]*B11[0-3][2] + + ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[0-3][7] -= A11[3][2]*B11[0-3][6] + + //unpacklow// + ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] + ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + + ///unpack high/// + ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[0][5] B11[1][5] B11[0][7] B11[1][7] + ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[2][5] B11[3][5] B11[2][7] B11[3][7] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm8 = _mm256_loadu_pd((double const *)(b11 + cs_b * 0)); //load B11[0-3][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b * 1)); //load B11[0-3][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //load B11[0-3][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //load B11[0-3][3] + ymm12 = _mm256_loadu_pd((double const *)(b11 + cs_b * 4)); //load B11[0-3][4] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b * 5)); //load B11[0-3][5] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b * 6)); //load B11[0-3][6] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b * 7)); //load B11[0-3][7] + //determine correct values to store + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x08); + ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x08); + ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x08); + ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x08); + ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x08); + ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x08); + ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x08); + ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x08); + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm0, ymm8, 0x30); + ymm1 = _mm256_permute2f128_pd(ymm1, ymm9, 0x30); + ymm2 = _mm256_permute2f128_pd(ymm2, ymm10, 0x30); + ymm3 = _mm256_permute2f128_pd(ymm3, ymm11, 0x30); + ymm4 = _mm256_permute2f128_pd(ymm4, ymm12, 0x30); + ymm5 = _mm256_permute2f128_pd(ymm5, ymm13, 0x30); + ymm6 = _mm256_permute2f128_pd(ymm6, ymm14, 0x30); + ymm7 = _mm256_permute2f128_pd(ymm7, ymm15, 0x30); + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x0E); + ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x0E); + ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x0E); + ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x0E); + ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x0E); + ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x0E); + ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x0E); + ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x0E); + } + + _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store(B11[0-3][4]) + _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store(B11[0-3][5]) + _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store(B11[0-3][6]) + _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store(B11[0-3][7]) + + } + } + + if((n & 4)) //implementation for remainder columns(when 'N' is a multiple of 4) + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM to be performed(in block of 4) + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b*3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a*2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[1][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[2][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[3][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B01[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B01[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B01[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]*B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]*B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]*B11[0][0-3] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]*B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]*B11[1][0-3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]*B11[2][0-3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b*2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[0-3][3]) + + } + if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + + k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) //looop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]* B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]* B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]* B11[0][0-3] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]* B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]* B11[1][0-3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]* B11[2][0-3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //load 4x4 block from b11 + ymm4 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][2] B11[3][3] + + //determine correct values to store + + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x08); + ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x08); + ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x08); + ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x08); + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm0, ymm4,0x30); + ymm1 = _mm256_permute2f128_pd(ymm1, ymm5,0x30); + ymm2 = _mm256_permute2f128_pd(ymm2, ymm6,0x30); + ymm3 = _mm256_permute2f128_pd(ymm3, ymm7,0x30); + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x0E); + ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x0E); + ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x0E); + ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x0E); + } + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) + + } + + n_remainder -= 4; + j += 4; + + } + + if(n_remainder) //implementation fo remaining columns(when 'N' is not a multiple of D_NR) + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha Value + + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + if(n_remainder == 3) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_broadcast_sd((double const *)&ones); + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const*)&ones); + } + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + } + + ///GEMM code ends/// + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0] * B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0] * B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0] * B11[0][0-3] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1] * B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1] * B11[1][0-3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2] * B11[2][0-3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + } + + } + if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + + k_iter = i / D_MR; //number of times GEMM operations to be performed + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha value + + ///GEMM for previously calculated values /// + + + //load 4x4 block from b11 + if(n_remainder == 3) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_broadcast_sd((double const *)&ones); + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[0][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[1][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[2][3] + + b01 += 1; //move to next row of B + + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[3][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm8 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] * alpha -= ymm4 + ymm9 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] * alpha -= ymm5 + ymm10 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] * alpha -= ymm6 + ymm11 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] * alpha -= ymm7 + + ///implement TRSM/// + //determine correct values to store + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x08); + ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x08); + ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x08); + ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x08); + + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm8, ymm0, 0x30); + ymm1 = _mm256_permute2f128_pd(ymm9, ymm1, 0x30); + ymm2 = _mm256_permute2f128_pd(ymm10, ymm2, 0x30); + ymm3 = _mm256_permute2f128_pd(ymm11, ymm3, 0x30); + + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x0E); + ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x0E); + ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x0E); + ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x0E); + } + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + } + + ///scalar code for trsm without alpha/// + dtrsm_small_AlXB_unitDiag(a11, b11, m_remainder, n_remainder, cs_a, cs_b); + } + } + return BLIS_SUCCESS; +} + + +/*implements TRSM for the case XA = alpha * B + *A is upper triangular, non-unit diagonal, no transpose + *dimensions: X:mxn A:nxn B: mxn + */ + +/* b11---> a01 ----> + ***************** *********** + *b01*b11* * * * * * * +b11 * * * * * **a01 * * a11 + | ***************** ********* | + | * * * * * *a11* * | + | * * * * * * * * | + v ***************** ****** v + * * * * * * * + * * * * * * * + ***************** * * + * + +*/ +static err_t bli_dtrsm_small_XAuB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if(max(m,n)>250 && (m/n) < 22) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction + { + for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction + { + a01 = L + j*cs_a; //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used in GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] + + //4th col + a11 += cs_a; + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] + + ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] + + ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j*cs_a; //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 2) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 1) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] + + //4th col + a11 += cs_a; + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] + + ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] + + ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + } + } + } + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = 0; (j+D_NR-1) a01 ----> + ***************** *********** + *b01*b11* * * * * * * +b11 * * * * * **a01 * * a11 + | ***************** ********* | + | * * * * * *a11* * | + | * * * * * * * * | + v ***************** ****** v + * * * * * * * + * * * * * * * + ***************** * * + * + +*/ + +static err_t bli_dtrsm_small_XAuB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if((max(m,n)>380) && (m/n)<22) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction + { + for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction + { + a01 = L + j*cs_a; //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used in GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] + + //4th col + a11 += cs_a; + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j*cs_a; //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 2) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 1) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] + + //4th col + a11 += cs_a; + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] + + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + } + } + } + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = 0; (j+D_NR-1) a01 ----> + ***************** *********** + *b01*b11* * * * * * * +b11 * * * * * **a01 * * a11 + | ***************** ********* | + | * * * * * *a11* * | + | * * * * * * * * | + v ***************** ****** v + * * * * * * * + * * * * * * * + ***************** * * + * + +*/ +static err_t bli_dtrsm_small_XAltB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if(max(m,n) > 250) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction + { + for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction + { + a01 = L + j; //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used in GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] + + ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] + + ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j; //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 2) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 1) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] + + ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] + + ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + } + } + } + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = 0; (j+D_NR-1) a01 ----> + ***************** *********** + *b01*b11* * * * * * * +b11 * * * * * **a01 * * a11 + | ***************** ********* | + | * * * * * *a11* * | + | * * * * * * * * | + v ***************** ****** v + * * * * * * * + * * * * * * * + ***************** * * + * + +*/ +static err_t bli_dtrsm_small_XAltB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if(max(m,n) > 250) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction + { + for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction + { + a01 = L + j; //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used in GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j; //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 2) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 1) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + } + } + } + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = 0; (j+D_NR-1) 250) + return BLIS_NOT_YET_IMPLEMENTED; + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction + { + for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction + { + a01 = L + j*cs_a +(j+D_NR); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm11 = _mm256_mul_pd(ymm11, ymm0); + + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(row 3):FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + ymm10 = _mm256_mul_pd(ymm10, ymm0); + + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + ymm9 = _mm256_mul_pd(ymm9, ymm0); + + ymm13 = _mm256_mul_pd(ymm13, ymm0); + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1): FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + + + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j + D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] + } + if(n_remainder == 2) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] + } + if(n_remainder == 1) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm11 = _mm256_mul_pd(ymm11, ymm0); + + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(row 3):FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + ymm10 = _mm256_mul_pd(ymm10, ymm0); + + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + ymm9 = _mm256_mul_pd(ymm9, ymm0); + + ymm13 = _mm256_mul_pd(ymm13, ymm0); + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1): FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + } + } + if(i<0) + i += D_NR; + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a33 + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm3 = _mm256_mul_pd(ymm3, ymm15); + + //extract a22 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + ymm2 = _mm256_mul_pd(ymm2, ymm15); + + //extract a11 + ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + ymm1 = _mm256_mul_pd(ymm1, ymm15); + + //extract A00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + ymm0 = _mm256_mul_pd(ymm0, ymm15); + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointwr to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + if(n_remainder == 3) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + } + if(n_remainder == 2) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + } + if(n_remainder == 1) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + } + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm16); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm16); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm16); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm16); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + + ///GEMM processing stars/// + + for(k = 0; k < k_iter; k++) + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + + } + + ///GEMM code ends/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -= ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a33 + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm3 = _mm256_mul_pd(ymm3, ymm15); + + //extract a22 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + ymm2 = _mm256_mul_pd(ymm2, ymm15); + + //extract a11 + ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + ymm1 = _mm256_mul_pd(ymm1, ymm15); + + //extract A00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + ymm0 = _mm256_mul_pd(ymm0, ymm15); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) + } + + } + m_remainder -= 4; + i -= 4; + } +// if(i < 0) i = 0; + if(m_remainder) ///implementation for remainder rows + { + dtrsm_small_XAlB(L, B, AlphaVal, m_remainder, n, cs_a, cs_b); + } + return BLIS_SUCCESS; +} + +/*implements TRSM for the case XA = alpha * B + *A is lower triangular, unit-diagonal, no transpose + *dimensions: X:mxn A:nxn B: mxn + */ + +/* <---b11 <---a11 + ***************** * + *b01*b11* * * * * + ^ * * * * * ^ * * + | ***************** | ******* + | * * * * * | * * * + | * * * * * a01* * * +b10 ***************** ************* + * * * * * * * * * + * * * * * * * * * + ***************** ******************* + +*/ +static err_t bli_dtrsm_small_XAlB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if(max(m,n) > 250) + return BLIS_NOT_YET_IMPLEMENTED; + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction + { + for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction + { + a01 = L + j*cs_a +(j+D_NR); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //(row 3):FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + //(Row 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + //(Row 1): FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + + + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j + D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] + } + if(n_remainder == 2) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] + } + if(n_remainder == 1) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //(row 3):FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + //(Row 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + //(Row 1): FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + } + } + if(i<0) + i += D_NR; + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //store(B11[x][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointwr to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + if(n_remainder == 3) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + } + if(n_remainder == 2) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + } + if(n_remainder == 1) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + } + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm16); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm16); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm16); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm16); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + + ///GEMM processing stars/// + + for(k = 0; k < k_iter; k++) + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + + } + + ///GEMM code ends/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -= ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) + } + + } + m_remainder -= 4; + i -= 4; + } + if(m_remainder) + { + dtrsm_small_XAlB_unitDiag(L, B, AlphaVal, m_remainder, n, cs_a, cs_b); + } + return BLIS_SUCCESS; +} + + +/*implements TRSM for the case XA = alpha * B + *A is lower triangular, non-unit diagonal, no transpose + *dimensions: X:mxn A:nxn B: mxn + */ + +/* <---b11 <---a11 + ***************** * + *b01*b11* * * * * + ^ * * * * * ^ * * + | ***************** | ******* + | * * * * * | * * * + | * * * * * a01* * * +b10 ***************** ************* + * * * * * * * * * + * * * * * * * * * + ***************** ******************* + +*/ +static err_t bli_dtrsm_small_XAutB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if(max(m,n) > 200) + return BLIS_NOT_YET_IMPLEMENTED; + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction + { + for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction + { + a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + ymm7 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm0 = _mm256_div_pd(ymm7, ymm0); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + //extract a33 + ymm7 = _mm256_permute_pd(ymm0, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm11 = _mm256_mul_pd(ymm11, ymm7); + + ymm15 = _mm256_mul_pd(ymm15, ymm7); + + //extract a22 + ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + //(Row 3): FMA operations + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + ymm10 = _mm256_mul_pd(ymm10, ymm7); + + ymm14 = _mm256_mul_pd(ymm14, ymm7); + + //extract a11 + ymm7 = _mm256_permute_pd(ymm0, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + ymm9 = _mm256_mul_pd(ymm9, ymm7); + + ymm13 = _mm256_mul_pd(ymm13, ymm7); + + //extract A00 + ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + ymm8 = _mm256_mul_pd(ymm8, ymm7); + + ymm12 = _mm256_mul_pd(ymm12, ymm7); + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[x][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + + a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] + } + if(n_remainder == 2) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] + } + if(n_remainder == 1) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + ymm7 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm0 = _mm256_div_pd(ymm7, ymm0); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + //extract a33 + ymm7 = _mm256_permute_pd(ymm0, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm11 = _mm256_mul_pd(ymm11, ymm7); + + ymm15 = _mm256_mul_pd(ymm15, ymm7); + + //extract a22 + ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + //(Row 3): FMA operations + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + ymm10 = _mm256_mul_pd(ymm10, ymm7); + + ymm14 = _mm256_mul_pd(ymm14, ymm7); + + //extract a11 + ymm7 = _mm256_permute_pd(ymm0, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + ymm9 = _mm256_mul_pd(ymm9, ymm7); + + ymm13 = _mm256_mul_pd(ymm13, ymm7); + + //extract A00 + ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + ymm8 = _mm256_mul_pd(ymm8, ymm7); + + ymm12 = _mm256_mul_pd(ymm12, ymm7); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + } + } + if(i<0) + i += D_NR; + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction + { + a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR*cs_a; //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a33 + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm3 = _mm256_mul_pd(ymm3, ymm15); + + //extract a22 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + ymm2 = _mm256_mul_pd(ymm2, ymm15); + + //extract a11 + ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + ymm1 = _mm256_mul_pd(ymm1, ymm15); + + //extract A00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + ymm0 = _mm256_mul_pd(ymm0, ymm15); + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) + { + + a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + if(n_remainder == 3) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + } + if(n_remainder == 2) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + } + if(n_remainder == 1) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + } + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a33 + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm3 = _mm256_mul_pd(ymm3, ymm15); + + //extract a22 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + ymm2 = _mm256_mul_pd(ymm2, ymm15); + + //extract a11 + ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + ymm1 = _mm256_mul_pd(ymm1, ymm15); + + //extract A00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + ymm0 = _mm256_mul_pd(ymm0, ymm15); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) + } + + } + m_remainder -= 4; + i -= 4; + } + if(m_remainder) ///implementation for remainder rows + { + dtrsm_small_XAutB(L, B, AlphaVal, m_remainder, n, cs_a, cs_b); + } + return BLIS_SUCCESS; +} + +/*implements TRSM for the case XA = alpha * B + *A is lower triangular, unit-diagonal, no transpose + *dimensions: X:mxn A:nxn B: mxn + */ + +/* <---b11 <---a11 + ***************** * + *b01*b11* * * * * + ^ * * * * * ^ * * + | ***************** | ******* + | * * * * * | * * * + | * * * * * a01* * * +b10 ***************** ************* + * * * * * * * * * + * * * * * * * * * + ***************** ******************* + +*/ +static err_t bli_dtrsm_small_XAutB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if(max(m,n) > 200) + return BLIS_NOT_YET_IMPLEMENTED; + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction + { + for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction + { + a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + //(Row 3): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + //(Row 3): FMA operations + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + //(ROW 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + //(Row 1):FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[x][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + + a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] + } + if(n_remainder == 2) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] + } + if(n_remainder == 1) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + //(Row 3): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + //(Row 3): FMA operations + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + //(ROW 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + //(Row 1):FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + + } + } + if(i<0) + i += D_NR; + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction + { + a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR*cs_a; //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) + { + + a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + if(n_remainder == 3) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + } + if(n_remainder == 2) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + } + if(n_remainder == 1) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + } + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) + } + + } + m_remainder -= 4; + i -= 4; + } + if(m_remainder) + { + dtrsm_small_XAutB_unitDiag(a->buffer, b->buffer,AlphaVal, m_remainder, n, cs_a, cs_b); + } + return BLIS_SUCCESS; } @@ -658,6 +12515,7 @@ static err_t bli_dtrsm_small_AlXB ( * AX = Alpha*B, Single precision, A: lower triangular * This kernel implementation supports matrices A and B such that m is equal to BLI_AlXB_M_SP and n is mutiple of 8 */ + static err_t bli_strsm_small_AlXB ( side_t side, obj_t* AlphaObj, @@ -776,101 +12634,6 @@ static err_t bli_strsm_small_AlXB ( return BLIS_SUCCESS; } -void trsm_block_c(float *ptr_l, float *ptr_b, int blk_height, int blk_width, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) -{ - int i, j, k, l; - float inv_l; - - inv_l = 1.0 / *ptr_l; - - for (j = 0; j < numCols_b; j += blk_width) - { - for (l = j; l < (j+blk_width); l++) - { - ptr_b[l*cs_b] = ptr_b[l*cs_b] * inv_l; - } - - for (i = 1; i < blk_height; i++) - { - for (l = j; l < (j+blk_width); l++) - { - for (k = 0; k < i; k++) - { - ptr_b[i*rs_b + l*cs_b] -= (ptr_b[k*rs_b + l*cs_b] * ptr_l[i*rs_l + k*cs_l]); - } - ptr_b[i*rs_b + l*cs_b] = ptr_b[i*rs_b + l*cs_b] / ptr_l[i*rs_l + i*cs_l]; - } - } - } -} - - -/* - * XA' = Alpha*B, Double precision, A:lower triangular - * This kernel implementation supports matrices A and B such that - * m and n are multiples of 4 and n less than or equal to BLI_XAltB_N_DP - */ - -static err_t bli_dtrsm_small_XAltB( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - - int m = bli_obj_length(a); // number of rows of matrix B - int n = bli_obj_length(b); // number of columns of matrix B - - int lda = bli_obj_col_stride(a); // column stride of A - int ldb = bli_obj_col_stride(b); // column stride of B - - int rsa = bli_obj_row_stride(a); // row stride of A - int rsb = bli_obj_row_stride(b); // row stride of B - - int i = 0; - int isUnitDiag = bli_obj_has_unit_diag(a); - - double alphaVal; - double *L = a->buffer; - double *B = b->buffer; - - if ((m&3) != 0 || (n&3) != 0) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - if ( n > BLI_XAltB_N_DP || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - alphaVal = *((double *)AlphaObj->buffer); - if (alphaVal != 1) - { - if (isUnitDiag == 0) - { - dtrsm_XAtB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - } - else - { - dtrsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - } - } - else - { - if (isUnitDiag == 0) - { - dtrsm_XAtB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - } - else - { - dtrsm_XAtB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - } - } - return BLIS_SUCCESS; - -} /* @@ -1004,902 +12767,7 @@ static err_t bli_strsm_small_AutXB( } return BLIS_SUCCESS; } -/* -* AX=B A=LOWER TRIANGULAR, NO TRANSPOSE, NON-UNITDIAGONAL -* ALPHA != 1; -*/ -static void blis_dtrsm_microkernel_alpha(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - double alphaVal - ) -{ - double ones = 1.0; - int j; - int cs_b_offset[2]; - double *ptr_b_dup; - __m256d mat_b_col[4]; - __m256d mat_b_rearr[4]; - __m256d mat_a_cols[4]; - __m256d mat_a_cols_rearr[10]; - __m256d mat_a_diag_inv[4]; - __m256d reciprocal_diags; - __m256d alphaReg; - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - - reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); - alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); - - //read first set of 4x4 block of B into registers - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); - - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //4th col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - numCols_b -= 4; // blk_width = 4 - - //compute reciprocals of L(i,i) and broadcast in registers - mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); - mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); - reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); - - for(j = 0;j < numCols_b; j += 4) - { - ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); - //extract a00 - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - - //Read next set of B columns - ptr_b += (cs_b+cs_b_offset[1]); - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - - } - //Last block trsm processing - - ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); - //extract a00 - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - -} -/* -*AX=B A=LOWER TRIANGULAR, NO TRANSPOSE, UNITDIAGONAL -*ALPHA != 1; -*/ -static void blis_dtrsm_microkernel_alpha_unitDiag(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - double alphaVal - ) -{ - - int j; - int cs_b_offset[2]; - double *ptr_b_dup; - - __m256d mat_b_col[4]; - __m256d mat_b_rearr[4]; - __m256d mat_a_cols[4]; - __m256d mat_a_cols_rearr[10]; - __m256d alphaReg; - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - - alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); - - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //4th col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - numCols_b -= 4; // blk_width = 4 - - for(j = 0;j < numCols_b; j += 4) - { - ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - - //Read next set of B columns - ptr_b += (cs_b+cs_b_offset[1]); - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - - } - //Last block trsm processing - - ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - -} -/* -*AX = B A= LOWERTRIANGULAR, NO TRANSPOSE, NON-UNITDIAGONAL -*ALPHA = 1 -*/ -static void blis_dtrsm_microkernel(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ) -{ - double ones = 1.0; - int j; - int cs_b_offset[2]; - double *ptr_b_dup; - - __m256d mat_b_col[4]; - __m256d mat_b_rearr[4]; - __m256d mat_a_cols[4]; - __m256d mat_a_cols_rearr[10]; - __m256d mat_a_diag_inv[4]; - __m256d reciprocal_diags; - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - - reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); - //row2 = (cs_l << 1); - //row4 = (cs_l << 2); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); - - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //4th col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - numCols_b -= 4; // blk_width = 4 - - //compute reciprocals of L(i,i) and broadcast in registers - mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); - mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); - reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); - - for(j = 0;j < numCols_b; j += 4) - { - ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - - //extract a00 - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - - //Read next set of B columns - ptr_b += (cs_b+cs_b_offset[1]); - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - - } - //Last block trsm processing - - ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - - //extract a00 - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); -} -/* -*AX = B A=LOWER TRIANGULAR, NO TRANSPOSE, UNITDIAGONAL -*ALPHA = 1 -*/ -static void blis_dtrsm_microkernel_unitDiag(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ) -{ - //double ones = 1.0; - int j; - int cs_b_offset[2]; - double *ptr_b_dup; - - __m256d mat_b_col[4]; - __m256d mat_b_rearr[4]; - __m256d mat_a_cols[4]; - __m256d mat_a_cols_rearr[10]; - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); - //row2 = (cs_l << 1); - //row4 = (cs_l << 2); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); - - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //4th col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - numCols_b -= 4; // blk_width = 4 - - - for(j = 0;j < numCols_b; j += 4) - { - ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - - //Read next set of B columns - ptr_b += (cs_b+cs_b_offset[1]); - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - } - //Last block trsm processing - - ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange high elements - - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); -} ///////////////////////////// AX=B /////////////////////////////// static void blis_strsm_microkernel_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) { @@ -4336,1137 +15204,272 @@ static void blis_strsm_microkernel(float *ptr_l, float *ptr_b, int numRows_lb, i _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); //end loop of cols } - -///////////////////////////////////// XA'=B functions //////////////////////////////// -static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - double alpha - ) - +static void blis_dtrsm_microkernel_alpha(double *ptr_l, + double *ptr_b, + int m, + int n, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ) { - - double ones = 1.0; - int i,i1,i2,i3,i4,j,k,l; - int cs_b_offset[3]; - int cs_l_offset[3]; + int j; + int n_remainder = n%4; + int cs_b_offset[2]; double *ptr_b_dup; - + double ones = 1.0; __m256d mat_b_col[4]; - __m256d mat_b_rearr[16][4]; - __m256d mat_a_cols_rearr[4]; - __m256d mat_a_blk_elems[16]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; __m256d mat_a_diag_inv[4]; - __m256d reciprocal_diags[2]; + __m256d reciprocal_diags; __m256d alphaReg; - reciprocal_diags[0] = _mm256_broadcast_sd((double const *)(&ones)); - alphaReg = _mm256_broadcast_sd((double const *)&alpha); - - // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - - //read diag elems of L 4x4 block - mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l); - mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[1]); cs_b_offset[0] = (cs_b << 1); cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - reciprocal_diags[1] = reciprocal_diags[0]; + reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); + alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); - //pack first 4 diags together - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 + //if(m % 4 == 0) + //{ + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - //Broadcast A21 to A31 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); + reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); - //Broadcast A32 to register - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - - /***************** first set of 4 cols of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 4) + for(j = 0;(j+3) < n; j += 4) { - /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A - //read 4x4 block of B into registers + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - mat_b_rearr[0][0] = _mm256_mul_pd(mat_b_rearr[0][0], alphaReg); - mat_b_rearr[1][0] = _mm256_mul_pd(mat_b_rearr[1][0], alphaReg); - mat_b_rearr[2][0] = _mm256_mul_pd(mat_b_rearr[2][0], alphaReg); - mat_b_rearr[3][0] = _mm256_mul_pd(mat_b_rearr[3][0], alphaReg); + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_pd(mat_b_rearr[0][0], mat_a_diag_inv[0]); + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_pd(mat_b_rearr[1][0], mat_a_diag_inv[1]); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_pd(mat_b_rearr[2][0], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_pd(mat_b_rearr[3][0], mat_a_diag_inv[3]); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - - i += 4; - ptr_b_dup += 4; - - } - - /***************** first set of 4 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width} - - for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row - { - ptr_l += 4; - ptr_b_dup += cs_b_offset[2]; - i1 += cs_b_offset[2]; - //printf("i1 = i3 = %g\n",*(ptr_l+i1)); - //Read next 4x4 block of A to get diag elements - i3 += cs_l_offset[2]; - mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l + i3); - mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[1]); - - //pack 4 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 - - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 - - //reciprocal of diagnal elements of A :- 0,1,2,3 - reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); - - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 4) - { - i = i1 + k; - mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - - mat_b_rearr[i2][0] = _mm256_mul_pd(mat_b_rearr[i2][0], alphaReg); - mat_b_rearr[i2][1] = _mm256_mul_pd(mat_b_rearr[i2][1], alphaReg); - mat_b_rearr[i2][2] = _mm256_mul_pd(mat_b_rearr[i2][2], alphaReg); - mat_b_rearr[i2][3] = _mm256_mul_pd(mat_b_rearr[i2][3], alphaReg); - i2++; - } - - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 4) // move across m - { - - //Broadcast A4,0 to A7,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - - //Broadcast A41 to A71 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); - - //Broadcast A4,2 to A7,2 to registers - mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); - - //Broadcast A4,3 to A7,3 to registers - mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); - - i += cs_l_offset[2]; - - for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); - - - i4 = k >> 2; - - //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - - - //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - //end loop of cols - - } - i2 += cs_b_offset[2]; - - } - - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //Broadcast A21 to A31 to registers - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //Broadcast A32 to A72 to registers - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - - k = 0; - for (i = 0; i < numCols_b; i+=4) - { - - - - /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[k][0] = _mm256_mul_pd(mat_b_rearr[k][0], mat_a_diag_inv[0]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[k][1] = _mm256_mul_pd(mat_b_rearr[k][1], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[k][2] = _mm256_mul_pd(mat_b_rearr[k][2], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[k][3] = _mm256_mul_pd(mat_b_rearr[k][3], mat_a_diag_inv[3]); - - //Store the computed B columns - - _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - - k++; - } - } - -} - -static void dtrsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - double alpha - ) - -{ - - int i,i1,i2,i3,i4,j,k,l; - int cs_b_offset[3]; - int cs_l_offset[3]; - double *ptr_b_dup; - - __m256d mat_b_col[4]; - __m256d mat_b_rearr[16][4]; - __m256d mat_a_blk_elems[16]; - __m256d alphaReg; - alphaReg = _mm256_broadcast_sd((double const *)&alpha); - - // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //Broadcast A21 to A31 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); - - //Broadcast A32 to register - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); - - /***************** first set of 4 cols of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 4) - { - /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A - //read 4x4 block of B into registers - - mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - - mat_b_rearr[0][0] = _mm256_mul_pd(mat_b_rearr[0][0], alphaReg); - mat_b_rearr[1][0] = _mm256_mul_pd(mat_b_rearr[1][0], alphaReg); - mat_b_rearr[2][0] = _mm256_mul_pd(mat_b_rearr[2][0], alphaReg); - mat_b_rearr[3][0] = _mm256_mul_pd(mat_b_rearr[3][0], alphaReg); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[0][0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[0][0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[0][0], mat_b_rearr[3][0]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[1][0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[1][0], mat_b_rearr[3][0]);//d = c - (a*b) - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_rearr[2][0], mat_b_rearr[3][0]);//d = c - (a*b) - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_b_rearr[0][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_rearr[1][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3][0]); - - i += 4; - ptr_b_dup += 4; - - } - - /***************** first set of 4 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width} - - for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row - { - ptr_l += 4; - ptr_b_dup += cs_b_offset[2]; - i1 += cs_b_offset[2]; - i3 += cs_l_offset[2]; - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 4) - { - i = i1 + k; - mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - - mat_b_rearr[i2][0] = _mm256_mul_pd(mat_b_rearr[i2][0], alphaReg); - mat_b_rearr[i2][1] = _mm256_mul_pd(mat_b_rearr[i2][1], alphaReg); - mat_b_rearr[i2][2] = _mm256_mul_pd(mat_b_rearr[i2][2], alphaReg); - mat_b_rearr[i2][3] = _mm256_mul_pd(mat_b_rearr[i2][3], alphaReg); - i2++; - } - - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 4) // move across m - { - - //Broadcast A4,0 to A7,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - - //Broadcast A41 to A71 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); - - //Broadcast A4,2 to A7,2 to registers - mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); - - //Broadcast A4,3 to A7,3 to registers - mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); - - i += cs_l_offset[2]; - - for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); - - - i4 = k >> 2; - - //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - - - //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - //end loop of cols - - } - i2 += cs_b_offset[2]; - - } - - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; - - //Broadcast A21 to A31 to registers - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; - - k = 0; - for (i = 0; i < numCols_b; i+=4) - { - - - - /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - - //Store the computed B columns - - _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - - k++; - } - - } - - -} - -static void dtrsm_XAtB_block_allSmallSizedMatrices_unitDiag(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ) - -{ - - int i,i1,i2,i3,i4,j,k,l; - int cs_b_offset[3]; - int cs_l_offset[3]; - double *ptr_b_dup; - - __m256d mat_b_col[4]; - __m256d mat_b_rearr[16][4]; - __m256d mat_a_blk_elems[16]; - - // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //Broadcast A21 to A31 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); - - //Broadcast A32 to register - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); - - /***************** first set of 4 cols of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 4) - { - /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A - //read 4x4 block of B into registers - - mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[0][0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[0][0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[0][0], mat_b_rearr[3][0]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[1][0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[1][0], mat_b_rearr[3][0]);//d = c - (a*b) - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_rearr[2][0], mat_b_rearr[3][0]);//d = c - (a*b) - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_b_rearr[0][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_rearr[1][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3][0]); - - i += 4; - ptr_b_dup += 4; - - } - - /***************** first set of 4 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width} - - for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row - { - ptr_l += 4; - ptr_b_dup += cs_b_offset[2]; - i1 += cs_b_offset[2]; - i3 += cs_l_offset[2]; - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 4) - { - i = i1 + k; - mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - - i2++; - } - - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 4) // move across m - { - - //Broadcast A4,0 to A7,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - - //Broadcast A41 to A71 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); - - //Broadcast A4,2 to A7,2 to registers - mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); - - //Broadcast A4,3 to A7,3 to registers - mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); - - i += cs_l_offset[2]; - - for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); - - - i4 = k >> 2; - - //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - - - //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - //end loop of cols - - } - i2 += cs_b_offset[2]; - - } - - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; - - //Broadcast A21 to A31 to registers - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; - - k = 0; - for (i = 0; i < numCols_b; i+=4) - { - - - - /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - - //Store the computed B columns - - _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - - k++; - } - - } - -} -static void dtrsm_XAtB_block_allSmallSizedMatrices(double *ptr_l, - double *ptr_b, - int numRows_lb, - int numCols_b, - int rs_l, - int rs_b, - int cs_l, - int cs_b - ) - -{ - - double ones = 1.0; - int i,i1,i2,i3,i4,j,k,l; - int cs_b_offset[3]; - int cs_l_offset[3]; - double *ptr_b_dup; - - __m256d mat_b_col[4]; - __m256d mat_b_rearr[16][4]; - __m256d mat_a_cols_rearr[4]; - __m256d mat_a_blk_elems[16]; - __m256d mat_a_diag_inv[4]; - __m256d reciprocal_diags[2]; - - reciprocal_diags[0] = _mm256_broadcast_sd((double const *)(&ones)); - - // ---> considering that the matrix size is multiple of 4 rows and 4 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - - //read diag elems of L 4x4 block - mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l); - mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + cs_l_offset[1]); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - - reciprocal_diags[1] = reciprocal_diags[0]; - - //pack first 4 diags together - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 - - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 - - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); - - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //Broadcast A21 to A31 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + 3)); - - //Broadcast A32 to register - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + 3)); - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - - /***************** first set of 4 cols of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 4) - { - /////////////////// Complete Upper 4x4 block trsm of B :- upper 4x4 block of B with upper 4x4 block of A - //read 4x4 block of B into registers - - mat_b_rearr[0][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_pd(mat_b_rearr[0][0], mat_a_diag_inv[0]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_pd(mat_b_rearr[1][0], mat_a_diag_inv[1]); + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_pd(mat_b_rearr[2][0], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3][0] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_pd(mat_b_rearr[3][0], mat_a_diag_inv[3]); - - //Store the computed B columns - _mm256_storeu_pd((double *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - - i += 4; - ptr_b_dup += 4; - - } - - /***************** first set of 4 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width} - - for (j = 4; j < numRows_lb; j += 4)//m :- 4x4 block row - { - ptr_l += 4; - ptr_b_dup += cs_b_offset[2]; - i1 += cs_b_offset[2]; - //printf("i1 = i3 = %g\n",*(ptr_l+i1)); - //Read next 4x4 block of A to get diag elements - i3 += cs_l_offset[2]; - mat_a_cols_rearr[0] = _mm256_loadu_pd((double const *)ptr_l + i3); - mat_a_cols_rearr[1] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_pd((double const *)ptr_l + i3 + cs_l_offset[1]); - - //pack 4 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0x0A);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_pd(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0x0A);//diag 2,3 - - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C);//diag 0,1,2,3 - - //reciprocal of diagnal elements of A :- 0,1,2,3 - reciprocal_diags[0] = _mm256_div_pd(reciprocal_diags[0], mat_a_diag_inv[0]); - - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 4) - { - i = i1 + k; - mat_b_rearr[i2][0] = _mm256_loadu_pd((double const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_pd((double const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1] + i)); - - i2++; - } - - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 4) // move across m - { - - //Broadcast A4,0 to A7,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - - //Broadcast A41 to A71 to registers - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[6] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[7] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l + i + 3)); - - //Broadcast A4,2 to A7,2 to registers - mat_a_blk_elems[8] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[9] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[0] + i + 3)); - - //Broadcast A4,3 to A7,3 to registers - mat_a_blk_elems[12] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[13] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[14] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[15] = _mm256_broadcast_sd((double const *)(ptr_l + cs_l_offset[1] + i + 3)); - - i += cs_l_offset[2]; - - for (k = 0; k < numCols_b; k += 4) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + i4 + cs_b_offset[1])); - - - i4 = k >> 2; - - //(Row4): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row5): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - - - //(Row6): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[8], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[9], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[10], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[11], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - //(Row7): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_pd(mat_a_blk_elems[12], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_pd(mat_a_blk_elems[13], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_pd(mat_a_blk_elems[14], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_pd(mat_a_blk_elems[15], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - //end loop of cols - - } - i2 += cs_b_offset[2]; - - } - - //Broadcast A10 to A30 to registers - mat_a_blk_elems[0] = _mm256_broadcast_sd((double const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //Broadcast A21 to A31 to registers - mat_a_blk_elems[3] = _mm256_broadcast_sd((double const *)(ptr_l + i + 2)); - mat_a_blk_elems[4] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags[0], 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[5] = _mm256_broadcast_sd((double const *)(ptr_l + i + 3)); - i += cs_l; //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags[0], 0x00); + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags[0], 0x0C); + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - k = 0; - for (i = 0; i < numCols_b; i+=4) - { + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - /////////////////// Complete Lower 4x4 block trsm of B :- lower 4x4 block of B with lower right 4x4 block of A + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[k][0] = _mm256_mul_pd(mat_b_rearr[k][0], mat_a_diag_inv[0]); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_pd(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[k][1] = _mm256_mul_pd(mat_b_rearr[k][1], mat_a_diag_inv[1]); + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_pd(mat_a_blk_elems[3], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[4], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[k][2] = _mm256_mul_pd(mat_b_rearr[k][2], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_pd(mat_a_blk_elems[5], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[k][3] = _mm256_mul_pd(mat_b_rearr[k][3], mat_a_diag_inv[3]); - - //Store the computed B columns - - _mm256_storeu_pd((double *)(ptr_b_dup + i), mat_b_rearr[k][0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - - k++; - } + } + ptr_b_dup = ptr_b; + if(n_remainder == 3) + { + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_broadcast_sd((double const *)&ones); + mat_b_col[3] = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_broadcast_sd((double const *)&ones); + mat_b_col[2] = _mm256_broadcast_sd((double const *)&ones); + mat_b_col[3] = _mm256_broadcast_sd((double const *)&ones); + } + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + //Store the computed B columns + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); } + //} + } + #if OPT_CACHE_BLOCKING_L1 //new intrinsic kernels static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) { @@ -15019,3 +25022,4 @@ static void trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, ///////////////////loop ends ///////////////////// } #endif +#endif diff --git a/kernels/zen2/3/bli_trsm_small.c b/kernels/zen2/3/bli_trsm_small.c new file mode 100644 index 000000000..346df3eca --- /dev/null +++ b/kernels/zen2/3/bli_trsm_small.c @@ -0,0 +1,25023 @@ +/* + +BLIS +An object-based framework for developing high-performance BLAS-like +libraries. + +Copyright (C) 2018, Advanced Micro Devices, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +- Neither the name of The University of Texas at Austin nor the names +of its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +#include "blis.h" +#ifdef BLIS_ENABLE_SMALL_MATRIX_TRSM +#include "immintrin.h" +#define GEMM_BLK_V1 8 //Block size to perform gemm and apply trsm +#define GEMM_ACCUM_A 1 //Peform B1=B1-(B0*A0) operation instead of B1'=(B0*A0) and then B1=B1-B1' +#define OPT_CACHE_BLOCKING_L1 1 //Perform trsm block-wise in blocks of GEMM_BLK_V1 instead of all columns of B together. +#define REARRANGE_SHFL 0 //Rearrange operations using blend or shuffle +#define BLI_AlXB_M_SP 16 +//#define BLI_AlXB_M_DP 16 +#define BLI_XAltB_N_SP 128 +#define BLI_AutXB_M_SP 64 +#define BLI_AutXB_N_SP 128 +#define max(a,b) a>b?a:b +#define min(a,b) a 0; k--) + { + double lkk_inv = 1.0/A[k+k*lda]; + for(i = M-1; i+1 > 0; i--) + { + B[i+k*ldb] *= lkk_inv; + for(j = k-1; j+1 > 0; j--) + { + B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is lower-triangular, unit-diagonal, no transpose + *Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAlB_unitDiag( + double *A, + double *B, + double alpha, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(i = 0 ; i < M; i++) + for(j = 0; j < N; j++) + B[i+j*ldb] *= alpha; + + for(k = N-1; k+1 > 0; k--) + { + for(i = M-1; i+1 > 0; i--) + { + for(j = k-1; j+1 > 0; j--) + { + B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + *A is upper-triangular, non-unit-diagonal, A is transposed + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAutB ( + double *A, + double *B, + double alpha, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(i = 0; i < M; i++) + for(j = 0; j < N; j++) + B[i+j*ldb] *=alpha; + + for(k = N-1; k+1 > 0; k--) + { + double lkk_inv = 1.0/A[k+k*lda]; + for(i = M-1; i+1 > 0; i--) + { + B[i+k*ldb] *= lkk_inv; + for(j = k-1; j+1 > 0; j--) + { + B[i+j*ldb] -= B[i+k*ldb] * A[j+k*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is upper-triangular, unit-diagonal, A has to be transposed + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAutB_unitDiag( + double *A, + double *B, + double alpha, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(i = 0; i< M; i++) + for(j = 0; j< N; j++) + B[i+j*ldb] *= alpha; + + for(i = M-1; i+1 > 0; i--) + { + for(j = N-1; j+1 > 0; j--) + { + for(k = j-1; k+1 > 0; k--) + { + B[i+k*ldb] -= B[i+j*ldb] * A[k+j*lda]; + + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is lower-triangular, non-unit-diagonal, A has to be transposed + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAltB ( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(k = 0; k < N; k++) + { + double lkk_inv = 1.0/A[k+k*lda]; + for(i = 0; i < M; i++) + { + B[i+k*ldb] *= lkk_inv; + for(j = k+1; j < N; j++) + { + B[i+j*ldb] -= B[i+k*ldb] * A[j+k*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for XA = alpha * B + * A is lower-triangular, unit-diagonal, A has to be transposed + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAltB_unitDiag( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(k = 0; k < N; k++) + { + for(i = 0; i < M; i++) + { + for(j = k+1; j < N; j++) + { + B[i+j*ldb] -= B[i+k*ldb] * A[j+k*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM scalar code for the case XA = alpha * B + * A is upper-triangular, unit-diagonal, no transpose + * Dimensions: X:mxn A:nxn B:mxn + */ +static err_t dtrsm_small_XAuB_unitDiag ( + double *A, + double *B, + dim_t M, + dim_t N, + dim_t lda, + dim_t ldb +) +{ + + dim_t i, j, k; + + for(k = 0; k < N; k++) + { + for(i = 0; i < M; i++) + { + for(j = k+1; j < N; j++) + { + B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; + } + } + } +return BLIS_SUCCESS; +} + +/* TRSM for the case AX = alpha * B, Double precision + * A is lower-triangular, no-transpose, non-unit diagonal + * dimensions A: mxm X: mxn B: mxn + + b01---> + * ***************** + ** * * * * * + * * * * * * * + * * *b01* * * * + * * * * * * * +a10 ****** b11 ***************** + | * * * | * * * * * + | * * * | * * * * * + | *a10*a11* | *b11* * * * + v * * * v * * * * * + *********** ***************** + * * * * * * * * * + * * * * * * * * * + * * * * * * * * * + * * * * * * * * * + **************** ***************** + a11---> +*/ + +static err_t bli_dtrsm_small_AlXB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + dim_t D_MR = 4; //size of block along 'M' dimpension + dim_t D_NR = 8; //size of block along 'N' dimension + + dim_t m = bli_obj_length(b); // number of rows of matrix B + dim_t n = bli_obj_width(b); // number of columns of matrix B + + if(max(m,n) > 90) + return BLIS_NOT_YET_IMPLEMENTED; + + dim_t m_remainder = m % D_MR; //number of remainder rows + dim_t n_remainder = n % D_NR; //number of remainder columns + + dim_t cs_a = bli_obj_col_stride(a); // column stride of A + dim_t cs_b = bli_obj_col_stride(b); // column stride of B + + dim_t i, j, k; //loop variables + dim_t k_iter; //number of times GEMM to be performed + + double AlphaVal = *(double *)AlphaObj->buffer; //value of alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a10, *a11, *b01, *b11; //pointers that point to blocks for GEMM and TRSM + double *ptr_b01_dup; + + double ones = 1.0; + + //scratch registers + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + + + for(j = 0; j+D_NR-1 < n; j += D_NR) //loop along 'N' dimension + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' dimension + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM to be performed(in blocks of 4x4) + + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + ///GEMM code begins/// + + for(k = 0; k< k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm16 = _mm256_loadu_pd((double const *)(a10));//A10[0][0] A10[1][0] A10[2][0] A10[3][0] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a));//A10[0][1] A10[1][1] A10[2][1] A10[3][1] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2));//A10[0][2] A10[1][2] A10[2][2] A10[3][2] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][4]*A10[3][2]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][5]*A10[3][2]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][6]*A10[3][2]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][7]*A10[3][2]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3));//A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[3][0] B01[3][0]*A10[2][3] B01[3][0]*A10[3][0]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[3][1]*A10[0][3] B01[3][1]*A10[3][0] B01[3][1]*A10[2][3] B01[3][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[3][2]*A10[0][3] B01[3][2]*A10[3][0] B01[3][2]*A10[2][3] B01[3][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[3][3]*A10[0][3] B01[3][3]*A10[3][0] B01[3][3]*A10[2][3] B01[3][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[3][4]*A10[0][3] B01[3][4]*A10[3][0] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[3][5]*A10[0][3] B01[3][5]*A10[3][0] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[3][6]*A10[0][3] B01[3][6]*A10[3][0] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[3][7]*A10[0][3] B01[3][7]*A10[3][0] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to calculate next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to calculate next block of B for GEMM + } + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha + + ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] * alpha -= B01[0-3][0] + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] * alpha -= B01[0-3][1] + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] * alpha -= B01[0-3][2] + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] * alpha -= B01[0-3][3] + ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] * alpha -= B01[0-3][4] + ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] * alpha -= B01[0-3][5] + ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] * alpha -= B01[0-3][6] + ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] * alpha -= B01[0-3][7] + + ///implement TRSM/// + + ///transpose of B11// + ///unpacklow/// + ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[2][4] B11[2][5] + ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[2][6] B11[2][7] + + //rearrange low elements + ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] + ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[1][4] B11[1][5] B11[3][4] B11[3][5] + ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[1][6] B11[1][7] B11[3][6] B11[3][7] + + //rearrange high elements + ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] + ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] + + ymm0 = _mm256_broadcast_sd((double const *)&ones); + + //broadcast diagonal elements of A11 + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] + ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] + + ymm5 = _mm256_unpacklo_pd(ymm1, ymm2); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] + ymm6 = _mm256_unpacklo_pd(ymm3, ymm4); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] + + ymm5 = _mm256_blend_pd(ymm5, ymm6, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm0 = _mm256_div_pd(ymm0, ymm5); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2] + + //extract a00 + ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + + //(Row 0): perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm1); //B11[0-3][0] /= A11[0][0] + ymm12 = _mm256_mul_pd(ymm12, ymm1); //B11[0-3][4] /= A11[0][0] + + //extract a11 + ymm1 = _mm256_permute_pd(ymm0, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] + + a11 += cs_a; + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= A11[1][0] * B11[0-3][0] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= A11[2][0] * B11[0-3][0] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= A11[3][0] * B11[0-3][0] + + ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= A11[1][0] * B11[0-3][4] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= A11[2][0] * B11[0-3][4] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= A11[3][0] * B11[0-3][4] + + ymm9 = _mm256_mul_pd(ymm9, ymm1); //B11[0-3][1] /= A11[1][1] + ymm13 = _mm256_mul_pd(ymm13, ymm1); //B11[0-3][5] /= A11[1][1] + + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] + + a11 += cs_a; + + //extract a22 + ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A110[][0] 1/A11[2][2] 1/A11[2][2] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(ROw2): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] + + ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] + + //perform mul operation + ymm10 = _mm256_mul_pd(ymm10, ymm1); //B11[0-3][2] /= A11[2][2] + ymm14 = _mm256_mul_pd(ymm14, ymm1); //B11[0-3][6] /= A11[2][2] + + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] + + a11 += cs_a; + + //extract a33 + ymm1 = _mm256_permute_pd(ymm0, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11);//1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(ROw2): FMA operations + ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[3][0-3] -= A11[3][2] * B11[0-3][2] + + ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[7][0-3] -= A11[3][2] * B11[0-3][6] + + //perform mul operation + ymm11 = _mm256_mul_pd(ymm11, ymm1); //B11[0-3][3] /= A11[3][3] + ymm15 = _mm256_mul_pd(ymm15, ymm1); //B11[0-3][7] /= A11[3][3] + + //unpacklow// + ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] + ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + + ///unpack high/// + ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[4][1] B11[5][1] B11[4][3] B11[5][3] + ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[6][1] B11[7][1] B11[6][3] B11[7][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store B11[0][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store B11[1][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store B11[2][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store B11[3][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store B11[4][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store B11[5][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store B11[6][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store B11[7][0-3] + } + + if(m_remainder) //implementation for reamainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM operation to be done(in blocks of 4x4) + + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + ///GEMM code Begins/// + for(k = 0; k< k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm16 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0] ) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm16 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 1)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] + + b01 += 1; //move to next row of B01 + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] //A10[1][2] A10[2][2] A10[3][2] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][0]*A10[3][2]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][1]*A10[3][2]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][2]*A10[3][2]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm8 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm8 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm8 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm8 += (B01[3][0]*A10[0][3] B01[3][4]*A10[1][3] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm8 += (B01[3][1]*A10[0][3] B01[3][5]*A10[1][3] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm8 += (B01[3][2]*A10[0][3] B01[3][6]*A10[1][3] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm8 += (B01[3][3]*A10[0][3] B01[3][7]*A10[1][3] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + } + + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value + + ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] *alpha -= B01[0-3][0] + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] *alpha -= B01[0-3][1] + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] *alpha -= B01[0-3][2] + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] *alpha -= B01[0-3][3] + ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] *alpha -= B01[0-3][4] + ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] *alpha -= B01[0-3][5] + ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] *alpha -= B01[0-3][6] + ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] *alpha -= B01[0-3][7] + + ///implement TRSM/// + + ///unpacklow/// + ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[1][4] B11[1][5] + ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[1][6] B11[1][7] + + //rearrange low elements + ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] + ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[5][0] B11[5][1] B11[7][0] B11[7][1] + ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[5][2] B11[5][3] B11[7][2] B11[7][3] + + //rearrange high elements + ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] + ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] + + + ymm0 = _mm256_broadcast_sd((double const *)&ones); + + //broadcast diagonal elements of A11 + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] + ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] + + ymm5 = _mm256_unpacklo_pd(ymm1, ymm2); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] + ymm6 = _mm256_unpacklo_pd(ymm3, ymm4); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] + + ymm5 = _mm256_blend_pd(ymm5, ymm6, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm0 = _mm256_div_pd(ymm0, ymm5); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a00 + ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + + //(Row 0): perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm1); //B11[0-3][0] /= A11[0][0] + ymm12 = _mm256_mul_pd(ymm12, ymm1); //B11[0-3][4] /= A11[0][0] + + //extract a11 + ymm1 = _mm256_permute_pd(ymm0, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] + + a11 += cs_a; + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= B11[0-3][0]*A11[1][0] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= B11[0-3][0]*A11[2][0] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= B11[0-3][0]*A11[3][0] + + ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= B11[0-3][4]*A11[1][4] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= B11[0-3][4]*A11[2][4] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= B11[0-3][4]*A11[3][4] + + ymm9 = _mm256_mul_pd(ymm9, ymm1); //B11[0-3][1] /= A11[1][1] + ymm13 = _mm256_mul_pd(ymm13, ymm1); //B11[0-3][5] /= A11[1][1] + + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] + + a11 += cs_a; + + //extract a22 + ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(ROw2): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] + + ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] + + //perform mul operation + ymm10 = _mm256_mul_pd(ymm10, ymm1); //B11[0-3][2] /=A11[2][2] + ymm14 = _mm256_mul_pd(ymm14, ymm1); //B11[0-3][6] /= A11[2][2] + + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] + + a11 += cs_a; + + //extract a33 + ymm1 = _mm256_permute_pd(ymm0, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(ROw2): FMA operations + ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[0-3][3] -= A11[3][2]*B11[0-3][2] + + ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[0-3][7] -= A11[3][2]*B11[0-3][6] + + //perform mul operation + ymm11 = _mm256_mul_pd(ymm11, ymm1); //B11[0-3][3] /= A11[3][3] + ymm15 = _mm256_mul_pd(ymm15, ymm1); //B11[0-3][7] /= A11[3][3] + + //unpacklow// + ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] + ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + + ///unpack high/// + ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[0][5] B11[1][5] B11[0][7] B11[1][7] + ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[2][5] B11[3][5] B11[2][7] B11[3][7] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm8 = _mm256_loadu_pd((double const *)(b11 + cs_b * 0)); //load B11[0-3][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b * 1)); //load B11[0-3][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //load B11[0-3][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //load B11[0-3][3] + ymm12 = _mm256_loadu_pd((double const *)(b11 + cs_b * 4)); //load B11[0-3][4] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b * 5)); //load B11[0-3][5] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b * 6)); //load B11[0-3][6] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b * 7)); //load B11[0-3][7] + //determine correct values to store + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x08); + ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x08); + ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x08); + ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x08); + ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x08); + ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x08); + ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x08); + ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x08); + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm0, ymm8, 0x30); + ymm1 = _mm256_permute2f128_pd(ymm1, ymm9, 0x30); + ymm2 = _mm256_permute2f128_pd(ymm2, ymm10, 0x30); + ymm3 = _mm256_permute2f128_pd(ymm3, ymm11, 0x30); + ymm4 = _mm256_permute2f128_pd(ymm4, ymm12, 0x30); + ymm5 = _mm256_permute2f128_pd(ymm5, ymm13, 0x30); + ymm6 = _mm256_permute2f128_pd(ymm6, ymm14, 0x30); + ymm7 = _mm256_permute2f128_pd(ymm7, ymm15, 0x30); + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x0E); + ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x0E); + ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x0E); + ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x0E); + ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x0E); + ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x0E); + ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x0E); + ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x0E); + } + + _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store(B11[0-3][4]) + _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store(B11[0-3][5]) + _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store(B11[0-3][6]) + _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store(B11[0-3][7]) + + } + } + + if((n & 4)) //implementation for remainder columns(when 'N' is a multiple of 4) + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM to be performed(in block of 4) + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b*3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a*2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[1][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[2][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[3][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B01[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B01[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B01[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + //compute reciprocals of L(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[0][0] A11[2][2] A11[2][2] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[1][1] A11[1][1] A11[3][3] A11[3][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + ymm4 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm4); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] +/* + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); +*/ + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] +/* + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); +*/ + //extract a00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm4 = _mm256_mul_pd(ymm4, ymm15); //B11[0][0-3] /= A11[0][0] + + //extract diag a11 from a + ymm15 = _mm256_permute_pd(ymm14, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]*B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]*B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]*B11[0][0-3] + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm15); //B11[1][0-3] /= A11[1][1] + + + //extract diag a22 from a + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]*B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]*B11[1][0-3] + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + ymm11 = _mm256_mul_pd(ymm11, ymm15); //B11[2][0-3] /= A11[2][2] + + //extract diag a33 from a + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]*B11[2][0-3] + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + ymm13 = _mm256_mul_pd(ymm13, ymm15); //B11[3][0-3] /= A11[3][3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b*2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[0-3][3]) + + } + if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + + k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) //looop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + //compute reciprocals of L(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + ymm4 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm4); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] +/* + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); +*/ + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] +/* + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); +*/ + //extract a00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00);//1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm4 = _mm256_mul_pd(ymm4, ymm15); //B11[0][0-3] /= A11[0][0] + + //extract diag a11 from a + ymm15 = _mm256_permute_pd(ymm14, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[][] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]* B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]* B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]* B11[0][0-3] + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm15); //B11[1][0-3] /= A11[1][1] + + + //extract diag a22 from a + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]* B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]* B11[1][0-3] + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + ymm11 = _mm256_mul_pd(ymm11, ymm15); //B11[2][0-3] /= A11[2][2] + + //extract diag a33 from a + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]* B11[2][0-3] + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + ymm13 = _mm256_mul_pd(ymm13, ymm15); //B11[3][0-3] /= A11[3][3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //load 4x4 block from b11 + ymm4 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][2] B11[3][3] + + //determine correct values to store + + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x08); + ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x08); + ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x08); + ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x08); + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm0, ymm4,0x30); + ymm1 = _mm256_permute2f128_pd(ymm1, ymm5,0x30); + ymm2 = _mm256_permute2f128_pd(ymm2, ymm6,0x30); + ymm3 = _mm256_permute2f128_pd(ymm3, ymm7,0x30); + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x0E); + ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x0E); + ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x0E); + ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x0E); + } + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) + + } + + n_remainder -= 4; + j += 4; + + } + + if(n_remainder) //implementation fo remaining columns(when 'N' is not a multiple of D_NR) + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha Value + + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + if(n_remainder == 3) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_broadcast_sd((double const *)&ones); + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const*)&ones); + } + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + } + + ///GEMM code ends/// + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + //compute reciprocals of L(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + ymm4 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm4); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] +/* + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); +*/ + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] +/* + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); +*/ + //extract a00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + ymm4 = _mm256_mul_pd(ymm4, ymm15); //B11[0][0-3] /= A11[0][0] + + //extract diag a11 from a + ymm15 = _mm256_permute_pd(ymm14, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0] * B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0] * B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0] * B11[0][0-3] + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + ymm8 = _mm256_mul_pd(ymm8, ymm15); //B11[1][0-3] /= A11[1][1] + + + //extract diag a22 from a + ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1] * B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1] * B11[1][0-3] + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + ymm11 = _mm256_mul_pd(ymm11, ymm15); //B11[2][0-3] /= A11[2][2] + + //extract diag a33 from a + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2] * B11[2][0-3] + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + ymm13 = _mm256_mul_pd(ymm13, ymm15); //B11[3][0-3] /= A11[3][3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + } + + } + if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + + k_iter = i / D_MR; //number of times GEMM operations to be performed + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha value + + ///GEMM for previously calculated values /// + + + //load 4x4 block from b11 + if(n_remainder == 3) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_broadcast_sd((double const *)&ones); + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[0][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[1][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[2][3] + + b01 += 1; //move to next row of B + + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[3][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm8 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] * alpha -= ymm4 + ymm9 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] * alpha -= ymm5 + ymm10 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] * alpha -= ymm6 + ymm11 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] * alpha -= ymm7 + + ///implement TRSM/// + //determine correct values to store + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x08); + ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x08); + ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x08); + ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x08); + + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm8, ymm0, 0x30); + ymm1 = _mm256_permute2f128_pd(ymm9, ymm1, 0x30); + ymm2 = _mm256_permute2f128_pd(ymm10, ymm2, 0x30); + ymm3 = _mm256_permute2f128_pd(ymm11, ymm3, 0x30); + + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x0E); + ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x0E); + ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x0E); + ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x0E); + } + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + } + + ///scalar code for trsm without alpha/// + dtrsm_small_AlXB(a11, b11, m_remainder, n_remainder, cs_a, cs_b); + } + } + return BLIS_SUCCESS; +} + +/* TRSM for the case AX = alpha * B, Double precision + * A is lower-triangular, no-transpose, unit diagonal + * dimensions A: mxm X: mxn B: mxn + + b01---> + * ***************** + ** * * * * * + * * * * * * * + * * *b01* * * * + * * * * * * * +a10 ****** b11 ***************** + | * * * | * * * * * + | * * * | * * * * * + | *a10*a11* | *b11* * * * + v * * * v * * * * * + *********** ***************** + * * * * * * * * * + * * * * * * * * * + * * * * * * * * * + * * * * * * * * * + **************** ***************** + a11---> +*/ + +static err_t bli_dtrsm_small_AlXB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + + dim_t D_MR = 4; //size of block along 'M' dimpension + dim_t D_NR = 8; //size of block along 'N' dimension + + dim_t m = bli_obj_length(b); // number of rows of matrix B + dim_t n = bli_obj_width(b); // number of columns of matrix B + + if(max(m,n) > 90) + return BLIS_NOT_YET_IMPLEMENTED; + + dim_t m_remainder = m % D_MR; //number of remainder rows + dim_t n_remainder = n % D_NR; //number of remainder columns + + dim_t cs_a = bli_obj_col_stride(a); // column stride of A + dim_t cs_b = bli_obj_col_stride(b); // column stride of B + + dim_t i, j, k; //loop variables + dim_t k_iter; //number of times GEMM to be performed + + double AlphaVal = *(double *)AlphaObj->buffer; //value of alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a10, *a11, *b01, *b11; //pointers that point to blocks for GEMM and TRSM + double *ptr_b01_dup; + + double ones = 1.0; + + //scratch registers + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + + + for(j = 0; j+D_NR-1 < n; j += D_NR) //loop along 'N' dimension + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' dimension + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM to be performed(in blocks of 4x4) + + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + ///GEMM code begins/// + + for(k = 0; k< k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm16 = _mm256_loadu_pd((double const *)(a10));//A10[0][0] A10[1][0] A10[2][0] A10[3][0] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a));//A10[0][1] A10[1][1] A10[2][1] A10[3][1] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2));//A10[0][2] A10[1][2] A10[2][2] A10[3][2] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][4]*A10[3][2]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][5]*A10[3][2]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][6]*A10[3][2]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][7]*A10[3][2]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3));//A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] + + b01 += 1; //mobe to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[3][0] B01[3][0]*A10[2][3] B01[3][0]*A10[3][0]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[3][1]*A10[0][3] B01[3][1]*A10[3][0] B01[3][1]*A10[2][3] B01[3][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[3][2]*A10[0][3] B01[3][2]*A10[3][0] B01[3][2]*A10[2][3] B01[3][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[3][3]*A10[0][3] B01[3][3]*A10[3][0] B01[3][3]*A10[2][3] B01[3][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[3][4]*A10[0][3] B01[3][4]*A10[3][0] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[3][5]*A10[0][3] B01[3][5]*A10[3][0] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[3][6]*A10[0][3] B01[3][6]*A10[3][0] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[3][7]*A10[0][3] B01[3][7]*A10[3][0] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to calculate next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to calculate next block of B for GEMM + } + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha + + ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] * alpha -= B01[0-3][0] + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] * alpha -= B01[0-3][1] + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] * alpha -= B01[0-3][2] + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] * alpha -= B01[0-3][3] + ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] * alpha -= B01[0-3][4] + ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] * alpha -= B01[0-3][5] + ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] * alpha -= B01[0-3][6] + ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] * alpha -= B01[0-3][7] + + ///implement TRSM/// + + ///transpose of B11// + ///unpacklow/// + ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[2][4] B11[2][5] + ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[2][6] B11[2][7] + + //rearrange low elements + ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] + ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[1][4] B11[1][5] B11[3][4] B11[3][5] + ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[1][6] B11[1][7] B11[3][6] B11[3][7] + + //rearrange high elements + ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] + ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] + + //broadcast diagonal elements of A11 + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] + ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] + + a11 += cs_a; + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= A11[1][0] * B11[0-3][0] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= A11[2][0] * B11[0-3][0] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= A11[3][0] * B11[0-3][0] + + ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= A11[1][0] * B11[0-3][4] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= A11[2][0] * B11[0-3][4] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= A11[3][0] * B11[0-3][4] + + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] + + a11 += cs_a; + + //(ROw2): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] + + ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] + + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] + + a11 += cs_a; + + //(ROw2): FMA operations + ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[3][0-3] -= A11[3][2] * B11[0-3][2] + + ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[7][0-3] -= A11[3][2] * B11[0-3][6] + + //unpacklow// + ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] + ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + + ///unpack high/// + ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[4][1] B11[5][1] B11[4][3] B11[5][3] + ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[6][1] B11[7][1] B11[6][3] B11[7][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store B11[0][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store B11[1][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store B11[2][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store B11[3][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store B11[4][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store B11[5][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store B11[6][0-3] + _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store B11[7][0-3] + } + + if(m_remainder) //implementation for reamainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM operation to be done(in blocks of 4x4) + + ymm8 = _mm256_setzero_pd(); + ymm9 = _mm256_setzero_pd(); + ymm10 = _mm256_setzero_pd(); + ymm11 = _mm256_setzero_pd(); + ymm12 = _mm256_setzero_pd(); + ymm13 = _mm256_setzero_pd(); + ymm14 = _mm256_setzero_pd(); + ymm15 = _mm256_setzero_pd(); + + ///GEMM code Begins/// + for(k = 0; k< k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm16 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0] ) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm16 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 1)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] + + b01 += 1; //move to next row of B01 + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] //A10[1][2] A10[2][2] A10[3][2] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][0]*A10[3][2]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][1]*A10[3][2]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][2]*A10[3][2]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] + ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] + ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] + ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] + + b01 += 1; //move to next row of B + + ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm8 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm8 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm8 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm8 += (B01[3][0]*A10[0][3] B01[3][4]*A10[1][3] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) + ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm8 += (B01[3][1]*A10[0][3] B01[3][5]*A10[1][3] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) + ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm8 += (B01[3][2]*A10[0][3] B01[3][6]*A10[1][3] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) + ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm8 += (B01[3][3]*A10[0][3] B01[3][7]*A10[1][3] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + } + + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value + + ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] *alpha -= B01[0-3][0] + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] *alpha -= B01[0-3][1] + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] *alpha -= B01[0-3][2] + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] *alpha -= B01[0-3][3] + ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] *alpha -= B01[0-3][4] + ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] *alpha -= B01[0-3][5] + ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] *alpha -= B01[0-3][6] + ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] *alpha -= B01[0-3][7] + + ///implement TRSM/// + + ///unpacklow/// + ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[1][4] B11[1][5] + ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[1][6] B11[1][7] + + //rearrange low elements + ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] + ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[5][0] B11[5][1] B11[7][0] B11[7][1] + ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[5][2] B11[5][3] B11[7][2] B11[7][3] + + //rearrange high elements + ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] + ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] + + //broadcast diagonal elements of A11 + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] + ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] + + a11 += cs_a; + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= B11[0-3][0]*A11[1][0] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= B11[0-3][0]*A11[2][0] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= B11[0-3][0]*A11[3][0] + + ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= B11[0-3][4]*A11[1][4] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= B11[0-3][4]*A11[2][4] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= B11[0-3][4]*A11[3][4] + + ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] + + a11 += cs_a; + + //(ROw2): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] + ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] + + ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] + ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] + + ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] + + a11 += cs_a; + + //(ROw2): FMA operations + ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[0-3][3] -= A11[3][2]*B11[0-3][2] + + ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[0-3][7] -= A11[3][2]*B11[0-3][6] + + //unpacklow// + ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] + ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] + ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] + + ///unpack high/// + ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[0][5] B11[1][5] B11[0][7] B11[1][7] + ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[2][5] B11[3][5] B11[2][7] B11[3][7] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] + ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] + + ymm8 = _mm256_loadu_pd((double const *)(b11 + cs_b * 0)); //load B11[0-3][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b * 1)); //load B11[0-3][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //load B11[0-3][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //load B11[0-3][3] + ymm12 = _mm256_loadu_pd((double const *)(b11 + cs_b * 4)); //load B11[0-3][4] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b * 5)); //load B11[0-3][5] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b * 6)); //load B11[0-3][6] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b * 7)); //load B11[0-3][7] + //determine correct values to store + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x08); + ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x08); + ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x08); + ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x08); + ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x08); + ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x08); + ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x08); + ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x08); + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm0, ymm8, 0x30); + ymm1 = _mm256_permute2f128_pd(ymm1, ymm9, 0x30); + ymm2 = _mm256_permute2f128_pd(ymm2, ymm10, 0x30); + ymm3 = _mm256_permute2f128_pd(ymm3, ymm11, 0x30); + ymm4 = _mm256_permute2f128_pd(ymm4, ymm12, 0x30); + ymm5 = _mm256_permute2f128_pd(ymm5, ymm13, 0x30); + ymm6 = _mm256_permute2f128_pd(ymm6, ymm14, 0x30); + ymm7 = _mm256_permute2f128_pd(ymm7, ymm15, 0x30); + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x0E); + ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x0E); + ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x0E); + ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x0E); + ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x0E); + ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x0E); + ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x0E); + ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x0E); + } + + _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store(B11[0-3][4]) + _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store(B11[0-3][5]) + _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store(B11[0-3][6]) + _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store(B11[0-3][7]) + + } + } + + if((n & 4)) //implementation for remainder columns(when 'N' is a multiple of 4) + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of times GEMM to be performed(in block of 4) + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b*3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a*2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[1][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[2][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[3][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B01[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B01[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B01[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]*B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]*B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]*B11[0][0-3] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]*B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]*B11[1][0-3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]*B11[2][0-3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b*2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[0-3][3]) + + } + if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + + k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) //looop for number of GEMM operations + { + ptr_b01_dup = b01; + + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]* B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]* B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]* B11[0][0-3] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]* B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]* B11[1][0-3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]* B11[2][0-3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //load 4x4 block from b11 + ymm4 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][2] B11[3][3] + + //determine correct values to store + + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x08); + ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x08); + ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x08); + ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x08); + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm0, ymm4,0x30); + ymm1 = _mm256_permute2f128_pd(ymm1, ymm5,0x30); + ymm2 = _mm256_permute2f128_pd(ymm2, ymm6,0x30); + ymm3 = _mm256_permute2f128_pd(ymm3, ymm7,0x30); + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x0E); + ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x0E); + ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x0E); + ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x0E); + } + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) + + } + + n_remainder -= 4; + j += 4; + + } + + if(n_remainder) //implementation fo remaining columns(when 'N' is not a multiple of D_NR) + { + for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha Value + + ///GEMM for previously calculated values /// + + //load 4x4 block from b11 + if(n_remainder == 3) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_broadcast_sd((double const *)&ones); + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const*)&ones); + } + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] + + b01 += 1; + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + } + + ///GEMM code ends/// + + ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 + ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 + ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 + ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 + + ///implement TRSM/// + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] + + ////unpacklow//// + ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] + ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] + + //rearrange low elements + ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] + ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] + + ////unpackhigh//// + ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] + ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] + + //rearrange high elements + ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] + ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0] * B11[0][0-3] + ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0] * B11[0][0-3] + ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0] * B11[0][0-3] + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1] * B11[1][0-3] + ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1] * B11[1][0-3] + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2] * B11[2][0-3] + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] + ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] + + //rearrange low elements + ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + + ////unpackhigh//// + ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] + + ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] + + //rearrange high elements + ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + } + + } + if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) + { + a10 = L +i; //pointer to block of A to be used for GEMM + a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM + b01 = B + j*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM + + + k_iter = i / D_MR; //number of times GEMM operations to be performed + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha value + + ///GEMM for previously calculated values /// + + + //load 4x4 block from b11 + if(n_remainder == 3) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_broadcast_sd((double const *)&ones); + ymm2 = _mm256_broadcast_sd((double const *)&ones); + ymm3 = _mm256_broadcast_sd((double const *)&ones); + } + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_b01_dup = b01; + ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[0][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[1][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[2][3] + + b01 += 1; //move to next row of B + + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) + + ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[3][3] + + b01 += 1; //move to next row of B + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) + + a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM + b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM + + } + + ymm8 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] * alpha -= ymm4 + ymm9 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] * alpha -= ymm5 + ymm10 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] * alpha -= ymm6 + ymm11 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] * alpha -= ymm7 + + ///implement TRSM/// + //determine correct values to store + if(m_remainder == 3) + { + ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x08); + ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x08); + ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x08); + ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x08); + + } + if(m_remainder == 2) + { + ymm0 = _mm256_permute2f128_pd(ymm8, ymm0, 0x30); + ymm1 = _mm256_permute2f128_pd(ymm9, ymm1, 0x30); + ymm2 = _mm256_permute2f128_pd(ymm10, ymm2, 0x30); + ymm3 = _mm256_permute2f128_pd(ymm11, ymm3, 0x30); + + } + if(m_remainder == 1) + { + ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x0E); + ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x0E); + ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x0E); + ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x0E); + } + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) + } + + ///scalar code for trsm without alpha/// + dtrsm_small_AlXB_unitDiag(a11, b11, m_remainder, n_remainder, cs_a, cs_b); + } + } + return BLIS_SUCCESS; +} + + +/*implements TRSM for the case XA = alpha * B + *A is upper triangular, non-unit diagonal, no transpose + *dimensions: X:mxn A:nxn B: mxn + */ + +/* b11---> a01 ----> + ***************** *********** + *b01*b11* * * * * * * +b11 * * * * * **a01 * * a11 + | ***************** ********* | + | * * * * * *a11* * | + | * * * * * * * * | + v ***************** ****** v + * * * * * * * + * * * * * * * + ***************** * * + * + +*/ +static err_t bli_dtrsm_small_XAuB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if(max(m,n)>150 && (m/n) < 22) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction + { + for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction + { + a01 = L + j*cs_a; //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used in GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] + + //4th col + a11 += cs_a; + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] + + ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] + + ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j*cs_a; //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 2) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 1) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] + + //4th col + a11 += cs_a; + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] + + ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] + + ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + } + } + } + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = 0; (j+D_NR-1) a01 ----> + ***************** *********** + *b01*b11* * * * * * * +b11 * * * * * **a01 * * a11 + | ***************** ********* | + | * * * * * *a11* * | + | * * * * * * * * | + v ***************** ****** v + * * * * * * * + * * * * * * * + ***************** * * + * + +*/ + +static err_t bli_dtrsm_small_XAuB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if((max(m,n)>180) && (m/n)<22) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction + { + for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction + { + a01 = L + j*cs_a; //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used in GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] + + //4th col + a11 += cs_a; + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j*cs_a; //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 2) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 1) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] + + //4th col + a11 += cs_a; + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] + + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + } + } + } + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = 0; (j+D_NR-1) a01 ----> + ***************** *********** + *b01*b11* * * * * * * +b11 * * * * * **a01 * * a11 + | ***************** ********* | + | * * * * * *a11* * | + | * * * * * * * * | + v ***************** ****** v + * * * * * * * + * * * * * * * + ***************** * * + * + +*/ +static err_t bli_dtrsm_small_XAltB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if(max(m,n) > 120) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction + { + for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction + { + a01 = L + j; //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used in GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] + + ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] + + ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j; //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 2) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 1) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] + + ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] + + ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] + + ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + } + } + } + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = 0; (j+D_NR-1) a01 ----> + ***************** *********** + *b01*b11* * * * * * * +b11 * * * * * **a01 * * a11 + | ***************** ********* | + | * * * * * *a11* * | + | * * * * * * * * | + v ***************** ****** v + * * * * * * * + * * * * * * * + ***************** * * + * + +*/ +static err_t bli_dtrsm_small_XAltB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if(max(m,n) > 120) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction + { + for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction + { + a01 = L + j; //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used in GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j; //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 2) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + if(n_remainder == 1) + { + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //(Row1): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] + ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] + ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] + + ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] + ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] + ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] + + //(Row2)FMA operations + ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] + ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] + + ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] + ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] + + //(Row3)FMA operations + ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] + + ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + } + } + } + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = 0; (j+D_NR-1) 120) + return BLIS_NOT_YET_IMPLEMENTED; + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction + { + for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction + { + a01 = L + j*cs_a +(j+D_NR); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm11 = _mm256_mul_pd(ymm11, ymm0); + + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(row 3):FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + ymm10 = _mm256_mul_pd(ymm10, ymm0); + + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + ymm9 = _mm256_mul_pd(ymm9, ymm0); + + ymm13 = _mm256_mul_pd(ymm13, ymm0); + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1): FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + + + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j + D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] + } + if(n_remainder == 2) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] + } + if(n_remainder == 1) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + //compute reciprocals of L(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //extract a33 + ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm11 = _mm256_mul_pd(ymm11, ymm0); + + ymm15 = _mm256_mul_pd(ymm15, ymm0); + + //extract a22 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(row 3):FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + ymm10 = _mm256_mul_pd(ymm10, ymm0); + + ymm14 = _mm256_mul_pd(ymm14, ymm0); + + //extract a11 + ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(Row 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + ymm9 = _mm256_mul_pd(ymm9, ymm0); + + ymm13 = _mm256_mul_pd(ymm13, ymm0); + + //extract a00 + ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1): FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] + + ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + } + } + if(i<0) + i += D_NR; + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a33 + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm3 = _mm256_mul_pd(ymm3, ymm15); + + //extract a22 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + ymm2 = _mm256_mul_pd(ymm2, ymm15); + + //extract a11 + ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + ymm1 = _mm256_mul_pd(ymm1, ymm15); + + //extract A00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + ymm0 = _mm256_mul_pd(ymm0, ymm15); + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointwr to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + if(n_remainder == 3) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + } + if(n_remainder == 2) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + } + if(n_remainder == 1) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + } + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm16); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm16); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm16); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm16); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + + ///GEMM processing stars/// + + for(k = 0; k < k_iter; k++) + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + + } + + ///GEMM code ends/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -= ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a33 + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm3 = _mm256_mul_pd(ymm3, ymm15); + + //extract a22 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + ymm2 = _mm256_mul_pd(ymm2, ymm15); + + //extract a11 + ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + ymm1 = _mm256_mul_pd(ymm1, ymm15); + + //extract A00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + ymm0 = _mm256_mul_pd(ymm0, ymm15); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) + } + + } + m_remainder -= 4; + i -= 4; + } +// if(i < 0) i = 0; + if(m_remainder) ///implementation for remainder rows + { + dtrsm_small_XAlB(L, B, AlphaVal, m_remainder, n, cs_a, cs_b); + } + return BLIS_SUCCESS; +} + +/*implements TRSM for the case XA = alpha * B + *A is lower triangular, unit-diagonal, no transpose + *dimensions: X:mxn A:nxn B: mxn + */ + +/* <---b11 <---a11 + ***************** * + *b01*b11* * * * * + ^ * * * * * ^ * * + | ***************** | ******* + | * * * * * | * * * + | * * * * * a01* * * +b10 ***************** ************* + * * * * * * * * * + * * * * * * * * * + ***************** ******************* + +*/ +static err_t bli_dtrsm_small_XAlB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if(max(m,n) > 120) + return BLIS_NOT_YET_IMPLEMENTED; + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction + { + for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction + { + a01 = L + j*cs_a +(j+D_NR); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //(row 3):FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + //(Row 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + //(Row 1): FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) + + + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j + D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) + + ///load 4x4 block of b11 + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation begins/// + + for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + + //subtract the calculated GEMM block from current TRSM block + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] + } + if(n_remainder == 2) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] + } + if(n_remainder == 1) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + ymm7 = _mm256_broadcast_sd((double const *)(&ones)); + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + //2nd col + a11 += 1; + ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] + + //3rd col + a11 += 1; + ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] + ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] + + //4th col + a11 += 1; + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] + + + ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] + ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] + ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] + + //(row 3):FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + //(Row 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + //(Row 1): FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + } + } + if(i<0) + i += D_NR; + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //store(B11[x][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) + { + a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointwr to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + if(n_remainder == 3) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + } + if(n_remainder == 2) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + } + if(n_remainder == 1) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + } + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm16); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm16); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm16); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm16); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + + ///GEMM processing stars/// + + for(k = 0; k < k_iter; k++) + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] + + a01 += 1; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM + + } + + ///GEMM code ends/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -= ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] + ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] + + //2nd col + a11 += cs_a; + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] + + //3rd col + a11 += cs_a; + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] + ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] + + //4th col + a11 += cs_a; + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) + } + + } + m_remainder -= 4; + i -= 4; + } + if(m_remainder) + { + dtrsm_small_XAlB_unitDiag(L, B, AlphaVal, m_remainder, n, cs_a, cs_b); + } + return BLIS_SUCCESS; +} + + +/*implements TRSM for the case XA = alpha * B + *A is lower triangular, non-unit diagonal, no transpose + *dimensions: X:mxn A:nxn B: mxn + */ + +/* <---b11 <---a11 + ***************** * + *b01*b11* * * * * + ^ * * * * * ^ * * + | ***************** | ******* + | * * * * * | * * * + | * * * * * a01* * * +b10 ***************** ************* + * * * * * * * * * + * * * * * * * * * + ***************** ******************* + +*/ +static err_t bli_dtrsm_small_XAutB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if(max(m,n) > 120) + return BLIS_NOT_YET_IMPLEMENTED; + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction + { + for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction + { + a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + ymm7 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm0 = _mm256_div_pd(ymm7, ymm0); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + //extract a33 + ymm7 = _mm256_permute_pd(ymm0, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm11 = _mm256_mul_pd(ymm11, ymm7); + + ymm15 = _mm256_mul_pd(ymm15, ymm7); + + //extract a22 + ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + //(Row 3): FMA operations + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + ymm10 = _mm256_mul_pd(ymm10, ymm7); + + ymm14 = _mm256_mul_pd(ymm14, ymm7); + + //extract a11 + ymm7 = _mm256_permute_pd(ymm0, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + ymm9 = _mm256_mul_pd(ymm9, ymm7); + + ymm13 = _mm256_mul_pd(ymm13, ymm7); + + //extract A00 + ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + ymm8 = _mm256_mul_pd(ymm8, ymm7); + + ymm12 = _mm256_mul_pd(ymm12, ymm7); + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[x][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + + a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] + } + if(n_remainder == 2) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] + } + if(n_remainder == 1) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + ymm7 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm0 = _mm256_div_pd(ymm7, ymm0); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + //extract a33 + ymm7 = _mm256_permute_pd(ymm0, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm11 = _mm256_mul_pd(ymm11, ymm7); + + ymm15 = _mm256_mul_pd(ymm15, ymm7); + + //extract a22 + ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + //(Row 3): FMA operations + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + ymm10 = _mm256_mul_pd(ymm10, ymm7); + + ymm14 = _mm256_mul_pd(ymm14, ymm7); + + //extract a11 + ymm7 = _mm256_permute_pd(ymm0, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + ymm9 = _mm256_mul_pd(ymm9, ymm7); + + ymm13 = _mm256_mul_pd(ymm13, ymm7); + + //extract A00 + ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + ymm8 = _mm256_mul_pd(ymm8, ymm7); + + ymm12 = _mm256_mul_pd(ymm12, ymm7); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + } + } + if(i<0) + i += D_NR; + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction + { + a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR*cs_a; //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a33 + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm3 = _mm256_mul_pd(ymm3, ymm15); + + //extract a22 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + ymm2 = _mm256_mul_pd(ymm2, ymm15); + + //extract a11 + ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + ymm1 = _mm256_mul_pd(ymm1, ymm15); + + //extract A00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + ymm0 = _mm256_mul_pd(ymm0, ymm15); + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) + { + + a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + if(n_remainder == 3) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + } + if(n_remainder == 2) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + } + if(n_remainder == 1) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + } + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + ymm14 = _mm256_broadcast_sd((double const *)&ones); + + //compute reciprocals of A(i,i) and broadcast in registers + ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] + ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] + + ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] + ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] + + //extract a33 + ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) + + ymm3 = _mm256_mul_pd(ymm3, ymm15); + + //extract a22 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + ymm2 = _mm256_mul_pd(ymm2, ymm15); + + //extract a11 + ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + ymm1 = _mm256_mul_pd(ymm1, ymm15); + + //extract A00 + ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) + ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + ymm0 = _mm256_mul_pd(ymm0, ymm15); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) + } + + } + m_remainder -= 4; + i -= 4; + } + if(m_remainder) ///implementation for remainder rows + { + dtrsm_small_XAutB(L, B, AlphaVal, m_remainder, n, cs_a, cs_b); + } + return BLIS_SUCCESS; +} + +/*implements TRSM for the case XA = alpha * B + *A is lower triangular, unit-diagonal, no transpose + *dimensions: X:mxn A:nxn B: mxn + */ + +/* <---b11 <---a11 + ***************** * + *b01*b11* * * * * + ^ * * * * * ^ * * + | ***************** | ******* + | * * * * * | * * * + | * * * * * a01* * * +b10 ***************** ************* + * * * * * * * * * + * * * * * * * * * + ***************** ******************* + +*/ +static err_t bli_dtrsm_small_XAutB_unitDiag( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + dim_t D_MR = 8; //block dimension along the rows + dim_t D_NR = 4; //block dimension along the columns + + dim_t m = bli_obj_length(b); //number of rows + dim_t n = bli_obj_width(b); //number of columns + dim_t m_remainder = m % D_MR; //number of corner rows + dim_t n_remainder = n % D_NR; //number of corner columns + dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A + dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B + + if(max(m,n) > 120) + return BLIS_NOT_YET_IMPLEMENTED; + + dim_t i, j, k; //loop variablse + dim_t k_iter; //determines the number of GEMM operations to be done + dim_t cs_b_offset[2]; //pre-calculated strides + + double ones = 1.0; + + double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha + double *L = a->buffer; //pointer to matrix A + double *B = b->buffer; //pointer to matrix B + + double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks + double *ptr_a01_dup; + + cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; + cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; + + //ymm scratch reginsters + __m256d ymm0, ymm1, ymm2, ymm3; + __m256d ymm4, ymm5, ymm6, ymm7; + __m256d ymm8, ymm9, ymm10, ymm11; + __m256d ymm12, ymm13, ymm14, ymm15; + __m256d ymm16; + + for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction + { + for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction + { + a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] + ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + //(Row 3): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + //(Row 3): FMA operations + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + //(ROW 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + //(Row 1):FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + _mm256_storeu_pd((double *)b11, ymm8); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[x][3]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) + { + + a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM + b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) + + ymm0 = _mm256_setzero_pd(); + ymm1 = _mm256_setzero_pd(); + ymm2 = _mm256_setzero_pd(); + ymm3 = _mm256_setzero_pd(); + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //broadcast 1st row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row + + //load 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) + + //broadcast 3rd row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A01 + + //load next 8x2 block of B10 + ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) + ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) + ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) + ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) + + ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) + + //broadcast 4th row of A01 + ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A01 + + ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) + ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) + ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) + ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) + + ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code ends/// + + ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); + //load 8x4 block of B11 + if(n_remainder == 3) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] + ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] + } + if(n_remainder == 2) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] + ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] + ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] + } + if(n_remainder == 1) + { + ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] + ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] + ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] + ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] + ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] + ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] + ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] + ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] + } + + ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 + ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 + ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 + ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 + + ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 + ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 + ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 + ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + //1st col + ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + //(Row 3): FMA operations + ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); + ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); + + //(Row 3): FMA operations + ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); + ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); + + //(ROW 2): FMA operations + ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); + ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); + + ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); + ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); + + //(Row 1):FMA operations + ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); + + ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) + } + + } + } + if(i<0) + i += D_NR; + if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) + { + for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction + { + a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + D_NR*cs_a; //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) + + } + if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) + { + + a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM + a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM + b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM + b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM + + k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) + + ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha + ///GEMM for previous blocks /// + + ///load 4x4 block of b11 + if(n_remainder == 3) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + } + if(n_remainder == 2) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + } + if(n_remainder == 1) + { + ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] + ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] + ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] + ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] + } + + //multiply by alpha + ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha + ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha + ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha + ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha + + ymm4 = _mm256_setzero_pd(); + ymm5 = _mm256_setzero_pd(); + ymm6 = _mm256_setzero_pd(); + ymm7 = _mm256_setzero_pd(); + + ///GEMM implementation starts/// + + for(k = 0; k < k_iter; k++) //loop for number of GEMM operations + { + ptr_a01_dup = a01; + + //load 4x4 bblock of b10 + ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] + ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] + ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] + ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] + + //broadcast 1st row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) + + //broadcast 2nd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) + + //braodcast 3rd row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) + + //broadcast 4th row of A01 + ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] + ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] + ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] + ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] + + a01 += cs_a; //move to next row of A + + ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) + ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) + ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) + ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) + + + b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM + a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM + } + + ///GEMM code end/// + + ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 + ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 + ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 + ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 + + ///implement TRSM/// + + ///read 4x4 block of A11/// + + + //1st col + ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] + + a11 += cs_a; + + //2nd col + ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + + a11 += cs_a; + + //3rd col + ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + + a11 += cs_a; + + //4th col + ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] + ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] + ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] + ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] + + //(Row 3): FMA operations + ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); + ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); + + //(ROW 2): FMA operations + ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); + ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); + + //(Row 1):FMA operations + ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); + + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) + _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) + _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) + } + + } + m_remainder -= 4; + i -= 4; + } + if(m_remainder) + { + dtrsm_small_XAutB_unitDiag(a->buffer, b->buffer,AlphaVal, m_remainder, n, cs_a, cs_b); + } + return BLIS_SUCCESS; +} + + +/* + * AX = Alpha*B, Single precision, A: lower triangular + * This kernel implementation supports matrices A and B such that m is equal to BLI_AlXB_M_SP and n is mutiple of 8 + */ + +static err_t bli_strsm_small_AlXB ( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + obj_t alpha, beta; // gemm parameters + obj_t Ga, Gb, Gc; // for GEMM + int m = bli_obj_length(b); // number of rows of matrix B + int n = bli_obj_width(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int j; + int blk_size = 8; + int isUnitDiag = bli_obj_has_unit_diag(a); + + float alphaVal; + float *L = a->buffer; + float *B = b->buffer; + + if (m != BLI_AlXB_M_SP || (n&7) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); + + /* Small _GEMM preparation code */ + bli_obj_create( BLIS_FLOAT, 1, 1, 0, 0, &alpha ); + bli_obj_create( BLIS_FLOAT, 1, 1, 0, 0, &beta ); + + /* B = B - A*B */ + bli_setsc( -(1.0), 0.0, &alpha ); + bli_setsc( (1.0), 0.0, &beta ); + + + bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, blk_size, a->buffer, rsa, lda, &Ga); + bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, n, b->buffer, rsb, ldb, &Gb); + bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, n, b->buffer, rsb, ldb, &Gc); + + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Ga ); + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gb ); + bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gc ); + + //first block of trsm + Gb.buffer = (void*)(B + i); + + //trsm of first 8xn block + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + blis_strsm_microkernel_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_strsm_microkernel = blis_strsm_microkernel; + } + else + { + blis_strsm_microkernel_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; + } + bli_setsc( alphaVal, 0.0, &beta ); + } + else + { + if (isUnitDiag == 0) + { + blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_strsm_microkernel = blis_strsm_microkernel; + } + else + { + blis_strsm_microkernel_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; + } + } + + //gemm update + for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + { + Ga.buffer = (void*)(L + j + i*lda); + Gc.buffer = (void*)(B + j); + + bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + } + + //trsm of remaining blocks + for (i = blk_size; i < m; i += blk_size) + { + Gb.buffer = (void*)(B + i); + + fp_blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + + for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT + { + Ga.buffer = (void*)(L + j + i*lda); + Gc.buffer = (void*)(B + j); + + bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb + } + + } // End of for loop - i + + return BLIS_SUCCESS; +} + + + +/* + * XA' = Alpha*B, Single precision, A: lower triangular + * This kernel implementation supports matrices A and B such that + * m and n are multiples of 8 and n is less than or equal to BLI_XAltB_N_SP + */ +static err_t bli_strsm_small_XAltB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int m = bli_obj_length(a); // number of rows of matrix B + int n = bli_obj_length(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int isUnitDiag = bli_obj_has_unit_diag(a); + + float alphaVal; + float *L = a->buffer; + float *B = b->buffer; + + if ((m&7) != 0 || (n&7) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( n > BLI_XAltB_N_SP || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); + + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + trsm_XAtB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + else + { + trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + } + else + { + if (isUnitDiag == 0) + { + trsm_XAtB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + else + { + trsm_XAtB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + } + return BLIS_SUCCESS; +} + +/* + * A'X = Alpha*B, Single precision, A: upper triangular + * This kernel implementation supports matrices A and B such that + * m and n are multiples of 8, m is less than or equal to BLI_AutXB_M_SP and n is less than or equal to BLI_AutXB_N_SP + */ +static err_t bli_strsm_small_AutXB( + side_t side, + obj_t* AlphaObj, + obj_t* a, + obj_t* b, + cntx_t* cntx, + cntl_t* cntl + ) +{ + int m = bli_obj_width(a); // number of rows of matrix A (since At, so width is taken) + int n = bli_obj_width(b); // number of columns of matrix B + + int lda = bli_obj_col_stride(a); // column stride of A + int ldb = bli_obj_col_stride(b); // column stride of B + + int rsa = bli_obj_row_stride(a); // row stride of A + int rsb = bli_obj_row_stride(b); // row stride of B + + int i = 0; + int isUnitDiag = bli_obj_has_unit_diag(a); + + float alphaVal; + float *L = a->buffer; + float *B = b->buffer; + + if ((m&7) != 0 || (n&7) != 0) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + if ( m > BLI_AutXB_M_SP || n > BLI_AutXB_N_SP || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) + { + return BLIS_NOT_YET_IMPLEMENTED; + } + + alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); + + if (alphaVal != 1) + { + if (isUnitDiag == 0) + { + trsm_AutXB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + else + { + trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); + } + } + else + { + if (isUnitDiag == 0) + { + trsm_AutXB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + else + { + trsm_AutXB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); + } + } + return BLIS_SUCCESS; +} + +///////////////////////////// AX=B /////////////////////////////// +static void blis_strsm_microkernel_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) +{ + float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags; + __m256 alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols +} + +static void blis_strsm_microkernel_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) +{ + //float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags; + __m256 alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //8th col + //ptr_l += cs_l; + //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + + //end loop of cols +} + +static void blis_strsm_microkernel_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //8th col + //ptr_l += cs_l; + //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols +} + +static void blis_strsm_microkernel(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int j; + int cs_b_offset[6]; + //int row2, row4, row6; + float *ptr_b_dup; + + //70 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_cols[8]; + __m256 mat_a_cols_rearr[36]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); + //row2 = (cs_l << 1); + //row4 = (cs_l << 2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); + //row6 = row2 + row4; + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); + + //reciprocal_diags = _mm256_loadu_ps((float const *)ones); + + //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L + /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); + ptr_l += cs_l; + mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ + + //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers + //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. + //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); + mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); + mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); + mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); + mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //4rth col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //5th col + ptr_l += cs_l; + mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //6th col + ptr_l += cs_l; + mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + //7th col + ptr_l += cs_l; + mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + numCols_b -= 8; // blk_width = 8 + + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); + mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); + + //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); + //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); + + //reciprocal of diagnol elements + reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); + + //Start loop for cols of B to be processed in size of blk_width + for (j = 0; j < numCols_b; j += 8) + { + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Read next set of B columns + ptr_b += (cs_b + cs_b_offset[5]); + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols + } + + //Last block trsm processing + ptr_b_dup = ptr_b; + + /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ + + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); + mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); +#else + mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); + mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); + mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); + mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); + mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); + mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); + mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); + mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); + mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); + //end loop of cols +} +static void blis_dtrsm_microkernel_alpha(double *ptr_l, + double *ptr_b, + int m, + int n, + int rs_l, + int rs_b, + int cs_l, + int cs_b, + double alphaVal + ) +{ + int j; + int n_remainder = n%4; + int cs_b_offset[2]; + double *ptr_b_dup; + double ones = 1.0; + __m256d mat_b_col[4]; + __m256d mat_b_rearr[4]; + __m256d mat_a_cols[4]; + __m256d mat_a_cols_rearr[10]; + __m256d mat_a_diag_inv[4]; + __m256d reciprocal_diags; + __m256d alphaReg; + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + + reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); + alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); + + //if(m % 4 == 0) + //{ + //1st col + mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); + mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); + mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); + mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); + + //2nd col + ptr_l += cs_l; + mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); + mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //3rd col + ptr_l += cs_l; + mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); + mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + + //4th col + ptr_l += cs_l; + mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); + //compute reciprocals of L(i,i) and broadcast in registers + mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); + mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); + + mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); + reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); + + for(j = 0;(j+3) < n; j += 4) + { + ptr_b_dup = ptr_b; + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); + mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); + + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + + //Read next set of B columns + ptr_b += (cs_b+cs_b_offset[1]); + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); + + } + ptr_b_dup = ptr_b; + if(n_remainder == 3) + { + + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); + mat_b_col[3] = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 2) + { + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); + mat_b_col[2] = _mm256_broadcast_sd((double const *)&ones); + mat_b_col[3] = _mm256_broadcast_sd((double const *)&ones); + } + if(n_remainder == 1) + { + //read first set of 4x4 block of B into registers + mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); + mat_b_col[1] = _mm256_broadcast_sd((double const *)&ones); + mat_b_col[2] = _mm256_broadcast_sd((double const *)&ones); + mat_b_col[3] = _mm256_broadcast_sd((double const *)&ones); + } + /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ + ////unpacklow//// + mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); + //rearrange low elements + mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); + mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); + //rearrange high elements + mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); + mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); + //extract a00 + mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); + mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) + mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); + mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); + mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); + //--> Transpose and store results of columns of B block <--// + ////unpacklow//// + mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); + //rearrange low elements + mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); + mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); + //rearrange high elements + mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); + mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); + //Store the computed B columns + if(n_remainder == 3) + { + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); + } + if(n_remainder == 2) + { + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); + } + if(n_remainder == 1) + { + _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); + } + + //} + +} + +#if OPT_CACHE_BLOCKING_L1 //new intrinsic kernels +static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0) + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0) + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + i = i1 + r; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + ptr_l_dup = ptr_l; + i4 = i2 + r; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + i4 = k >> 3; + ptr_l_dup += cs_l; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); + ptr_l_dup += cs_l; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + } + i2 += cs_b_offset[6]; + i += cs_l_offset[6]; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + { + i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done + +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} +#else //rel 1.0 intrisic kernels (NOT OPT_CACHE_BLOCKING_L1) +static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + __m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + __m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); + mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); + mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); + mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); + mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += cs_l_offset[6]; + mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); + mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); + mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); + mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); + mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); + + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + //__m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + //(Row0) + mat_b_col[0] = mat_b_rearr[0][0]; + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): already done + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} + +static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + //float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[16][8]; + //__m256 mat_a_cols_rearr[8]; + __m256 mat_a_blk_elems[64]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); + mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); + mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); + mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); + mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); + mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); + mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); + mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); + + //(Row0) + mat_b_col[0] = mat_b_rearr[0][0]; + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) + mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) + mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) + mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) + mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) + mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) + mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); + + //i += cs_b_offset[6]; + //ptr_b_dup += cs_b_offset[6]; + i += 8; + ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += 8; + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += cs_b_offset[6]; + i1 += cs_b_offset[6]; + i3 += cs_l_offset[6]; + + i = 0; + i2 = 0; + for (k = 0; k < numCols_b; k += 8) + { + i = i1 + k; + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); + mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); + mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); + mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); + mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); + mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); + mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); + mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); + + i2++; + } + + i = 0; + i2 = 0; + for (l = 0; l < j; l += 8) // move across m + { + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); + + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); + + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); + mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); + mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); + mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); + mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); + + // _mm256_permute2f128_ps() + + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); + mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); + mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); + mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); + mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); + mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); + mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); + mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); + + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); + mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); + mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); + mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); + mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); + mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); + mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); + mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); + + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); + mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); + mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); + mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); + mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); + mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); + mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); + mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); + + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); + mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); + mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); + mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); + mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); + mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); + mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); + mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); + + i += cs_l_offset[6]; + + for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + { + /////////////////// Partial Lower 8x8 block trsm of B + + i4 = i2 + k; + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + i4 = k >> 3; + + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) + + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) + mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) + mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) + mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) + mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) + mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) + mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) + mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) + + //end loop of cols + } + i2 += cs_b_offset[6]; + } + + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + i += cs_l; + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); + + k = 0; + for (i = 0; i < numCols_b; i+=8) + { + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A + + //(Row0): already done + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) + + //////////////////////////////////////////////////////////////////////////////// + + //Store the computed B columns + + _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + } + + + } + ///////////////////loop ends ///////////////////// +} +#endif //OPT_CACHE_BLOCKING_L1 + +//////////////////////////// AutX=B /////////////////////// +static void trsm_AutXB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += 8; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ +#endif + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_AutXB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + float ones = 1.0; + int i, i1, i2, i3, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + __m256 mat_a_diag_inv[8]; + __m256 reciprocal_diags[2]; + __m256 alphaReg; + + reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + //read diag elems of L 16x16 block + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + + reciprocal_diags[1] = reciprocal_diags[0]; + + //pack first 8 diags together + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i3 = 0; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + //Read next 8x8 block of A to get diag elements + i3 += 8; + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); + + //pack 8 diags of A together + reciprocal_diags[0] = reciprocal_diags[1]; + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 + mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 + mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 + mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 + mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 + + //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 + reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + //extract diag a00 from a + mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); + //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); + + //extract diag a11 from a + mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); + //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); + + //extract diag a22 from a + mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); + //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); + + //extract diag a33 from a + mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); + //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); + + //extract diag a44 from a + mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); + mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); + //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); + + //extract diag a55 from a + mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); + mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); + //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); + + //extract diag a66 from a + mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); + mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); + //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); + + //extract diag a77 from a + mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); + mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); + //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_AutXB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) +{ + //float ones = 1.0; + int i, i1, i2, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + + //(Row0) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ +#endif + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done + +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} + +static void trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) +{ + //float ones = 1.0; + int i, i1, i2, i4, j, k, l, r; + int cs_b_offset[7]; + int cs_l_offset[7]; + float *ptr_b_dup, *ptr_l_dup; + + //57 number of ymm(256 bits) registers used + __m256 mat_b_col[8]; + __m256 mat_b_rearr[8]; + __m256 mat_a_blk_elems[8]; + //__m256 mat_a_diag_inv[8]; + //__m256 reciprocal_diags[2]; + __m256 alphaReg; + alphaReg = _mm256_broadcast_ss((float const *)&alpha); + + // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // + + //L matrix offsets + cs_l_offset[0] = (cs_l << 1); + cs_l_offset[1] = cs_l + cs_l_offset[0]; + cs_l_offset[2] = (cs_l << 2); + cs_l_offset[3] = cs_l + cs_l_offset[2]; + cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; + cs_l_offset[5] = cs_l + cs_l_offset[4]; + cs_l_offset[6] = (cs_l_offset[5] + cs_l); + + cs_b_offset[0] = (cs_b << 1); + cs_b_offset[1] = cs_b + cs_b_offset[0]; + cs_b_offset[2] = (cs_b << 2); + cs_b_offset[3] = cs_b + cs_b_offset[2]; + cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; + cs_b_offset[5] = cs_b + cs_b_offset[4]; + cs_b_offset[6] = (cs_b_offset[5] + cs_b); + +#if 0 + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); + + //Broadcast A21 to A71 to registers + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); + mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); + mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); + mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); + mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); + mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); + mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); + mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); + mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); + mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); + mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); + mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); + mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); + mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); + mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); + mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); + + //Broadcast A76 to register + mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); +#endif + + + /***************** first set of 8 rows of B processing starts *****************/ + ptr_b_dup = ptr_b; + i = 0; + for (j = 0; j < numCols_b; j += 8) + { + /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A + //read 8x8 block of B into registers + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); + + //(Row0) + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) + + + + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); + + i += cs_b_offset[6]; + ptr_b_dup += cs_b_offset[6]; + //i += 8; + //ptr_b_dup += 8; + } + + //c = 0; + /***************** first set of 8 cols of B processing done *****************/ + ptr_b_dup = ptr_b; + i1 = 0; + //Start loop for cols of B to be processed in size of blk_width + for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row + { + ptr_l += cs_l_offset[6]; + + + //ptr_b += j; + //ptr_b_dup += 8; + ptr_b_dup += 8; + i1 += 8; + i = i1; + i2 = 0; + + for (r = 0; r < numCols_b; r += GEMM_BLK_V1) + { +#if GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); + mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); + mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); + mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + + ////unpackhigh//// + mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); + mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); + mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); + mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + /* transpose steps end */ + + mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); + mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); + mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); + mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); + mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); + mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); + mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); + mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); +#endif + + //i = 0; + ptr_l_dup = ptr_l; + i4 = i2; + for (l = 0; l < j; l += 8) // move across m + { + //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) + //{ + /////////////////// Partial Lower 8x8 block trsm of B + //Read current 8 cols of B columns from specified 8x8 current-block of B + mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); + mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); + mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); + mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); + mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); + mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); + mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); + mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); + mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); + mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); + mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); + mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); +#else + mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); + mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); + mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); + mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); + mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); + mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); + /* transpose steps end */ + + //Broadcast A8,0 to A15,0 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i4 = k >> 3; + ptr_l_dup++; + +#if GEMM_ACCUM_A + //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); + mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); + mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); + mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); + mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); + mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); + mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); + mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,2 to A15,2 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,3 to A15,3 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,4 to A15,4 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,5 to A15,5 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,6 to A15,6 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A8,7 to A15,7 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + ptr_l_dup++; +#if GEMM_ACCUM_A + //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) + mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) +#endif + //end loop of cols + //} + //i2 += cs_b_offset[6]; + i4 += 8; + } + //trsm solve + + k = 0; + //for (i2 = 0; i2 < numCols_b; i2 += 8) + //{ + //i2 = i1 + r; + /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A +#if !GEMM_ACCUM_A + //Read 8 cols of B columns of Block-to-be-solved + mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); + mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); + mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); + mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); + mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); + mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); + mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); + mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); + mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); + mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); + mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); + mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); + mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); + mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); + mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); +#endif + //Broadcast A10 to A70 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); + mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); + //i += cs_l; + +#if GEMM_ACCUM_A + //(Row0): already done + +#else + mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); +#endif + +#if GEMM_ACCUM_A + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#else + mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); + mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); + mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); + mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); + mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); + mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); + mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); + + //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) + mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) +#endif + //Broadcast A21 to A71 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); + mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) + mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A32 to A72 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); + mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) + mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A43 to A73 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); + mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) + mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A54 to A74 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); + mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) + mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A65 to A75 to registers + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); + mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); + //i += cs_l; + + + + //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) + mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) + + //Broadcast A76 to register + mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); + + + + //(Row7): FMA operations of b7 with elements of index (7, 0) + mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) + + + + //////////////////////////////////////////////////////////////////////////////// + + /* transpose steps start */ + ////unpacklow//// + mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange low elements +#if REARRANGE_SHFL == 1 + mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); + mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); +#else + mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); + mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); + mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); + mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); + mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); + mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); +#endif + //Merge rearranged low elements into complete rows + mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); + mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); + mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); + mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); + + ////unpackhigh//// + mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); + mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); + mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); + mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); + + //Rearrange high elements +#if REARRANGE_SHFL == 1 + mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); + mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); +#else + mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); + mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); + mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); + mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); + mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); + mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); +#endif + + //Merge rearranged high elements into complete rows + mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); + mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); + mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); + mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); + /* transpose steps end */ + + //Store the computed B columns + _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); + _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); + _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); + //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); + k++; + //} + i += cs_b_offset[6]; + i2 += cs_b_offset[6]; + } + } //numRows of A + ///////////////////loop ends ///////////////////// +} +#endif From 9d93a4caa21402d3a90aac45d7a1603736c9fd63 Mon Sep 17 00:00:00 2001 From: prangana Date: Fri, 24 May 2019 17:59:13 +0530 Subject: [PATCH 43/53] update version 2.0 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 4b9fcbec1..cd5ac039d 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.5.1 +2.0 From ee123f535872510f77100d3d55a43d4ca56047d5 Mon Sep 17 00:00:00 2001 From: Meghana Date: Mon, 27 May 2019 15:36:44 +0530 Subject: [PATCH 44/53] Defined small matrix thresholds for TRSM for various cases for NAPLES and ROME Updated copyright information for kernels/zen/bli_trsm_small.c file Removed separate kernels for zen2 architecture Instead added threshold conditions in zen kernels both for ROME and NAPLES Change-Id: Ifd715731741d649b6ad16b123a86dbd6665d97e5 --- config/zen/bli_family_zen.h | 7 + config/zen2/bli_family_zen2.h | 5 + kernels/zen/3/bli_trsm_small.c | 122 +- kernels/zen2/3/bli_trsm_small.c | 25023 ------------------------------ 4 files changed, 111 insertions(+), 25046 deletions(-) delete mode 100644 kernels/zen2/3/bli_trsm_small.c diff --git a/config/zen/bli_family_zen.h b/config/zen/bli_family_zen.h index 5e1dc9bb2..67a0806c9 100644 --- a/config/zen/bli_family_zen.h +++ b/config/zen/bli_family_zen.h @@ -59,5 +59,12 @@ //This macro will enable BLIS DGEMM to choose block sizes for a single instance mode #define BLIS_ENABLE_SINGLE_INSTANCE_BLOCK_SIZES 0 + +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES 250 +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_ALXB_NAPLES 90 + +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_DIM_RATIO 22 + + //#endif diff --git a/config/zen2/bli_family_zen2.h b/config/zen2/bli_family_zen2.h index e2a622b31..3422450d8 100644 --- a/config/zen2/bli_family_zen2.h +++ b/config/zen2/bli_family_zen2.h @@ -60,6 +60,11 @@ #define BLIS_ENABLE_SMALL_MATRIX_ROME #define BLIS_SMALL_MATRIX_THRES_ROME 400 +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME 120 +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_ALXB_ROME 60 +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_XAUB_ROME 150 + +#define D_BLIS_SMALL_MATRIX_THRES_TRSM_DIM_RATIO 22 #endif diff --git a/kernels/zen/3/bli_trsm_small.c b/kernels/zen/3/bli_trsm_small.c index 21114829d..af84d0588 100644 --- a/kernels/zen/3/bli_trsm_small.c +++ b/kernels/zen/3/bli_trsm_small.c @@ -4,7 +4,7 @@ BLIS An object-based framework for developing high-performance BLAS-like libraries. -Copyright (C) 2018, Advanced Micro Devices, Inc. +Copyright (C) 2018-2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -34,19 +34,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "blis.h" #ifdef BLIS_ENABLE_SMALL_MATRIX_TRSM -#ifndef BLI_FAMILY_ZEN2_ #include "immintrin.h" #define GEMM_BLK_V1 8 //Block size to perform gemm and apply trsm #define GEMM_ACCUM_A 1 //Peform B1=B1-(B0*A0) operation instead of B1'=(B0*A0) and then B1=B1-B1' #define OPT_CACHE_BLOCKING_L1 1 //Perform trsm block-wise in blocks of GEMM_BLK_V1 instead of all columns of B together. #define REARRANGE_SHFL 0 //Rearrange operations using blend or shuffle #define BLI_AlXB_M_SP 16 -//#define BLI_AlXB_M_DP 16 #define BLI_XAltB_N_SP 128 #define BLI_AutXB_M_SP 64 #define BLI_AutXB_N_SP 128 -#define max(a,b) a>b?a:b -#define min(a,b) a 90) - return BLIS_NOT_YET_IMPLEMENTED; +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ALXB_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ALXB_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif dim_t m_remainder = m % D_MR; //number of remainder rows dim_t n_remainder = n % D_NR; //number of remainder columns @@ -2506,8 +2511,17 @@ static err_t bli_dtrsm_small_AlXB_unitDiag( dim_t m = bli_obj_length(b); // number of rows of matrix B dim_t n = bli_obj_width(b); // number of columns of matrix B - if(max(m,n) > 90) - return BLIS_NOT_YET_IMPLEMENTED; +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ALXB_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ALXB_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif dim_t m_remainder = m % D_MR; //number of remainder rows dim_t n_remainder = n % D_NR; //number of remainder columns @@ -3857,10 +3871,17 @@ static err_t bli_dtrsm_small_XAuB( dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - if(max(m,n)>250 && (m/n) < 22) +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n)>D_BLIS_SMALL_MATRIX_THRES_TRSM_XAUB_ROME && (m/n) < D_BLIS_SMALL_MATRIX_THRES_TRSM_DIM_RATIO) { return BLIS_NOT_YET_IMPLEMENTED; } +#else + if(bli_max(m,n)>D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES && (m/n) < D_BLIS_SMALL_MATRIX_THRES_TRSM_DIM_RATIO) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif dim_t i, j, k; //loop variablse dim_t k_iter; //determines the number of GEMM operations to be done @@ -5195,10 +5216,17 @@ static err_t bli_dtrsm_small_XAuB_unitDiag( dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - if((max(m,n)>380) && (m/n)<22) +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n)>D_BLIS_SMALL_MATRIX_THRES_TRSM_XAUB_ROME && (m/n) < D_BLIS_SMALL_MATRIX_THRES_TRSM_DIM_RATIO) { return BLIS_NOT_YET_IMPLEMENTED; } +#else + if(bli_max(m,n)>D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES && (m/n) < D_BLIS_SMALL_MATRIX_THRES_TRSM_DIM_RATIO) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif dim_t i, j, k; //loop variablse dim_t k_iter; //determines the number of GEMM operations to be done @@ -6373,10 +6401,17 @@ static err_t bli_dtrsm_small_XAltB( dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - if(max(m,n) > 250) +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME) { return BLIS_NOT_YET_IMPLEMENTED; } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif dim_t i, j, k; //loop variablse dim_t k_iter; //determines the number of GEMM operations to be done @@ -7718,11 +7753,17 @@ static err_t bli_dtrsm_small_XAltB_unitDiag( dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - if(max(m,n) > 250) +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME) { return BLIS_NOT_YET_IMPLEMENTED; } - +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif dim_t i, j, k; //loop variablse dim_t k_iter; //determines the number of GEMM operations to be done @@ -8886,8 +8927,17 @@ static err_t bli_dtrsm_small_XAlB( dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - if(max(m,n) > 250) - return BLIS_NOT_YET_IMPLEMENTED; +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif dim_t i, j, k; //loop variablse dim_t k_iter; //determines the number of GEMM operations to be done @@ -9868,8 +9918,17 @@ static err_t bli_dtrsm_small_XAlB_unitDiag( dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - if(max(m,n) > 250) - return BLIS_NOT_YET_IMPLEMENTED; +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif dim_t i, j, k; //loop variablse dim_t k_iter; //determines the number of GEMM operations to be done @@ -10709,8 +10768,17 @@ static err_t bli_dtrsm_small_XAutB( dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - if(max(m,n) > 200) - return BLIS_NOT_YET_IMPLEMENTED; +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif dim_t i, j, k; //loop variablse dim_t k_iter; //determines the number of GEMM operations to be done @@ -11702,8 +11770,17 @@ static err_t bli_dtrsm_small_XAutB_unitDiag( dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - if(max(m,n) > 200) - return BLIS_NOT_YET_IMPLEMENTED; +#ifdef BLIS_ENABLE_SMALL_MATRIX_ROME + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_ROME) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#else + if(bli_max(m,n) > D_BLIS_SMALL_MATRIX_THRES_TRSM_NAPLES) + { + return BLIS_NOT_YET_IMPLEMENTED; + } +#endif dim_t i, j, k; //loop variablse dim_t k_iter; //determines the number of GEMM operations to be done @@ -25022,4 +25099,3 @@ static void trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, ///////////////////loop ends ///////////////////// } #endif -#endif diff --git a/kernels/zen2/3/bli_trsm_small.c b/kernels/zen2/3/bli_trsm_small.c deleted file mode 100644 index 346df3eca..000000000 --- a/kernels/zen2/3/bli_trsm_small.c +++ /dev/null @@ -1,25023 +0,0 @@ -/* - -BLIS -An object-based framework for developing high-performance BLAS-like -libraries. - -Copyright (C) 2018, Advanced Micro Devices, Inc. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: -- Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -- Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. -- Neither the name of The University of Texas at Austin nor the names -of its contributors may be used to endorse or promote products -derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -*/ - -#include "blis.h" -#ifdef BLIS_ENABLE_SMALL_MATRIX_TRSM -#include "immintrin.h" -#define GEMM_BLK_V1 8 //Block size to perform gemm and apply trsm -#define GEMM_ACCUM_A 1 //Peform B1=B1-(B0*A0) operation instead of B1'=(B0*A0) and then B1=B1-B1' -#define OPT_CACHE_BLOCKING_L1 1 //Perform trsm block-wise in blocks of GEMM_BLK_V1 instead of all columns of B together. -#define REARRANGE_SHFL 0 //Rearrange operations using blend or shuffle -#define BLI_AlXB_M_SP 16 -//#define BLI_AlXB_M_DP 16 -#define BLI_XAltB_N_SP 128 -#define BLI_AutXB_M_SP 64 -#define BLI_AutXB_N_SP 128 -#define max(a,b) a>b?a:b -#define min(a,b) a 0; k--) - { - double lkk_inv = 1.0/A[k+k*lda]; - for(i = M-1; i+1 > 0; i--) - { - B[i+k*ldb] *= lkk_inv; - for(j = k-1; j+1 > 0; j--) - { - B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; - } - } - } -return BLIS_SUCCESS; -} - -/* TRSM scalar code for the case XA = alpha * B - * A is lower-triangular, unit-diagonal, no transpose - *Dimensions: X:mxn A:nxn B:mxn - */ -static err_t dtrsm_small_XAlB_unitDiag( - double *A, - double *B, - double alpha, - dim_t M, - dim_t N, - dim_t lda, - dim_t ldb -) -{ - - dim_t i, j, k; - - for(i = 0 ; i < M; i++) - for(j = 0; j < N; j++) - B[i+j*ldb] *= alpha; - - for(k = N-1; k+1 > 0; k--) - { - for(i = M-1; i+1 > 0; i--) - { - for(j = k-1; j+1 > 0; j--) - { - B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; - } - } - } -return BLIS_SUCCESS; -} - -/* TRSM scalar code for the case XA = alpha * B - *A is upper-triangular, non-unit-diagonal, A is transposed - * Dimensions: X:mxn A:nxn B:mxn - */ -static err_t dtrsm_small_XAutB ( - double *A, - double *B, - double alpha, - dim_t M, - dim_t N, - dim_t lda, - dim_t ldb -) -{ - - dim_t i, j, k; - - for(i = 0; i < M; i++) - for(j = 0; j < N; j++) - B[i+j*ldb] *=alpha; - - for(k = N-1; k+1 > 0; k--) - { - double lkk_inv = 1.0/A[k+k*lda]; - for(i = M-1; i+1 > 0; i--) - { - B[i+k*ldb] *= lkk_inv; - for(j = k-1; j+1 > 0; j--) - { - B[i+j*ldb] -= B[i+k*ldb] * A[j+k*lda]; - } - } - } -return BLIS_SUCCESS; -} - -/* TRSM scalar code for the case XA = alpha * B - * A is upper-triangular, unit-diagonal, A has to be transposed - * Dimensions: X:mxn A:nxn B:mxn - */ -static err_t dtrsm_small_XAutB_unitDiag( - double *A, - double *B, - double alpha, - dim_t M, - dim_t N, - dim_t lda, - dim_t ldb -) -{ - - dim_t i, j, k; - - for(i = 0; i< M; i++) - for(j = 0; j< N; j++) - B[i+j*ldb] *= alpha; - - for(i = M-1; i+1 > 0; i--) - { - for(j = N-1; j+1 > 0; j--) - { - for(k = j-1; k+1 > 0; k--) - { - B[i+k*ldb] -= B[i+j*ldb] * A[k+j*lda]; - - } - } - } -return BLIS_SUCCESS; -} - -/* TRSM scalar code for the case XA = alpha * B - * A is lower-triangular, non-unit-diagonal, A has to be transposed - * Dimensions: X:mxn A:nxn B:mxn - */ -static err_t dtrsm_small_XAltB ( - double *A, - double *B, - dim_t M, - dim_t N, - dim_t lda, - dim_t ldb -) -{ - - dim_t i, j, k; - - for(k = 0; k < N; k++) - { - double lkk_inv = 1.0/A[k+k*lda]; - for(i = 0; i < M; i++) - { - B[i+k*ldb] *= lkk_inv; - for(j = k+1; j < N; j++) - { - B[i+j*ldb] -= B[i+k*ldb] * A[j+k*lda]; - } - } - } -return BLIS_SUCCESS; -} - -/* TRSM scalar code for XA = alpha * B - * A is lower-triangular, unit-diagonal, A has to be transposed - * Dimensions: X:mxn A:nxn B:mxn - */ -static err_t dtrsm_small_XAltB_unitDiag( - double *A, - double *B, - dim_t M, - dim_t N, - dim_t lda, - dim_t ldb -) -{ - - dim_t i, j, k; - - for(k = 0; k < N; k++) - { - for(i = 0; i < M; i++) - { - for(j = k+1; j < N; j++) - { - B[i+j*ldb] -= B[i+k*ldb] * A[j+k*lda]; - } - } - } -return BLIS_SUCCESS; -} - -/* TRSM scalar code for the case XA = alpha * B - * A is upper-triangular, unit-diagonal, no transpose - * Dimensions: X:mxn A:nxn B:mxn - */ -static err_t dtrsm_small_XAuB_unitDiag ( - double *A, - double *B, - dim_t M, - dim_t N, - dim_t lda, - dim_t ldb -) -{ - - dim_t i, j, k; - - for(k = 0; k < N; k++) - { - for(i = 0; i < M; i++) - { - for(j = k+1; j < N; j++) - { - B[i+j*ldb] -= B[i+k*ldb] * A[k+j*lda]; - } - } - } -return BLIS_SUCCESS; -} - -/* TRSM for the case AX = alpha * B, Double precision - * A is lower-triangular, no-transpose, non-unit diagonal - * dimensions A: mxm X: mxn B: mxn - - b01---> - * ***************** - ** * * * * * - * * * * * * * - * * *b01* * * * - * * * * * * * -a10 ****** b11 ***************** - | * * * | * * * * * - | * * * | * * * * * - | *a10*a11* | *b11* * * * - v * * * v * * * * * - *********** ***************** - * * * * * * * * * - * * * * * * * * * - * * * * * * * * * - * * * * * * * * * - **************** ***************** - a11---> -*/ - -static err_t bli_dtrsm_small_AlXB( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - - dim_t D_MR = 4; //size of block along 'M' dimpension - dim_t D_NR = 8; //size of block along 'N' dimension - - dim_t m = bli_obj_length(b); // number of rows of matrix B - dim_t n = bli_obj_width(b); // number of columns of matrix B - - if(max(m,n) > 90) - return BLIS_NOT_YET_IMPLEMENTED; - - dim_t m_remainder = m % D_MR; //number of remainder rows - dim_t n_remainder = n % D_NR; //number of remainder columns - - dim_t cs_a = bli_obj_col_stride(a); // column stride of A - dim_t cs_b = bli_obj_col_stride(b); // column stride of B - - dim_t i, j, k; //loop variables - dim_t k_iter; //number of times GEMM to be performed - - double AlphaVal = *(double *)AlphaObj->buffer; //value of alpha - double *L = a->buffer; //pointer to matrix A - double *B = b->buffer; //pointer to matrix B - - double *a10, *a11, *b01, *b11; //pointers that point to blocks for GEMM and TRSM - double *ptr_b01_dup; - - double ones = 1.0; - - //scratch registers - __m256d ymm0, ymm1, ymm2, ymm3; - __m256d ymm4, ymm5, ymm6, ymm7; - __m256d ymm8, ymm9, ymm10, ymm11; - __m256d ymm12, ymm13, ymm14, ymm15; - __m256d ymm16; - - - - for(j = 0; j+D_NR-1 < n; j += D_NR) //loop along 'N' dimension - { - for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' dimension - { - a10 = L +i; //pointer to block of A to be used for GEMM - a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM - b01 = B + j*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM - - k_iter = i / D_MR; //number of times GEMM to be performed(in blocks of 4x4) - - ymm8 = _mm256_setzero_pd(); - ymm9 = _mm256_setzero_pd(); - ymm10 = _mm256_setzero_pd(); - ymm11 = _mm256_setzero_pd(); - ymm12 = _mm256_setzero_pd(); - ymm13 = _mm256_setzero_pd(); - ymm14 = _mm256_setzero_pd(); - ymm15 = _mm256_setzero_pd(); - - ///GEMM code begins/// - - for(k = 0; k< k_iter; k++) //loop for number of GEMM operations - { - ptr_b01_dup = b01; - - ymm16 = _mm256_loadu_pd((double const *)(a10));//A10[0][0] A10[1][0] A10[2][0] A10[3][0] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] - - b01 += 1; //mobe to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) - - ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a));//A10[0][1] A10[1][1] A10[2][1] A10[3][1] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] - - b01 += 1; //mobe to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) - - ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2));//A10[0][2] A10[1][2] A10[2][2] A10[3][2] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] - - b01 += 1; //mobe to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][4]*A10[3][2]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][5]*A10[3][2]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][6]*A10[3][2]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][7]*A10[3][2]) - - ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3));//A10[0][3] A10[1][3] A10[2][3] A10[3][3] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] - - b01 += 1; //mobe to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[3][0] B01[3][0]*A10[2][3] B01[3][0]*A10[3][0]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[3][1]*A10[0][3] B01[3][1]*A10[3][0] B01[3][1]*A10[2][3] B01[3][1]*A10[3][0]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[3][2]*A10[0][3] B01[3][2]*A10[3][0] B01[3][2]*A10[2][3] B01[3][2]*A10[3][0]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[3][3]*A10[0][3] B01[3][3]*A10[3][0] B01[3][3]*A10[2][3] B01[3][3]*A10[3][0]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[3][4]*A10[0][3] B01[3][4]*A10[3][0] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[3][5]*A10[0][3] B01[3][5]*A10[3][0] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[3][6]*A10[0][3] B01[3][6]*A10[3][0] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[3][7]*A10[0][3] B01[3][7]*A10[3][0] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) - - a10 += D_MR * cs_a; //pointer math to calculate next block of A for GEMM - b01 = ptr_b01_dup + D_MR; //pointer math to calculate next block of B for GEMM - } - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha - - ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] - ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] - ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] - ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] - - ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] * alpha -= B01[0-3][0] - ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] * alpha -= B01[0-3][1] - ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] * alpha -= B01[0-3][2] - ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] * alpha -= B01[0-3][3] - ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] * alpha -= B01[0-3][4] - ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] * alpha -= B01[0-3][5] - ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] * alpha -= B01[0-3][6] - ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] * alpha -= B01[0-3][7] - - ///implement TRSM/// - - ///transpose of B11// - ///unpacklow/// - ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] - ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] - - ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[2][4] B11[2][5] - ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[2][6] B11[2][7] - - //rearrange low elements - ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] - ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] - - ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] - ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] - - ////unpackhigh//// - ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] - ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] - - ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[1][4] B11[1][5] B11[3][4] B11[3][5] - ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[1][6] B11[1][7] B11[3][6] B11[3][7] - - //rearrange high elements - ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] - ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] - - ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] - ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] - - ymm0 = _mm256_broadcast_sd((double const *)&ones); - - //broadcast diagonal elements of A11 - ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] - ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] - - ymm5 = _mm256_unpacklo_pd(ymm1, ymm2); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] - ymm6 = _mm256_unpacklo_pd(ymm3, ymm4); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] - - ymm5 = _mm256_blend_pd(ymm5, ymm6, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm0 = _mm256_div_pd(ymm0, ymm5); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2] - - //extract a00 - ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] - ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] - - //(Row 0): perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - ymm8 = _mm256_mul_pd(ymm8, ymm1); //B11[0-3][0] /= A11[0][0] - ymm12 = _mm256_mul_pd(ymm12, ymm1); //B11[0-3][4] /= A11[0][0] - - //extract a11 - ymm1 = _mm256_permute_pd(ymm0, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] - ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] - - ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] - ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] - ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] - - a11 += cs_a; - - //(Row1): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= A11[1][0] * B11[0-3][0] - ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= A11[2][0] * B11[0-3][0] - ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= A11[3][0] * B11[0-3][0] - - ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= A11[1][0] * B11[0-3][4] - ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= A11[2][0] * B11[0-3][4] - ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= A11[3][0] * B11[0-3][4] - - ymm9 = _mm256_mul_pd(ymm9, ymm1); //B11[0-3][1] /= A11[1][1] - ymm13 = _mm256_mul_pd(ymm13, ymm1); //B11[0-3][5] /= A11[1][1] - - ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] - ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] - - a11 += cs_a; - - //extract a22 - ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A110[][0] 1/A11[2][2] 1/A11[2][2] - ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] - - //(ROw2): FMA operations - ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] - ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] - - ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] - ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] - - //perform mul operation - ymm10 = _mm256_mul_pd(ymm10, ymm1); //B11[0-3][2] /= A11[2][2] - ymm14 = _mm256_mul_pd(ymm14, ymm1); //B11[0-3][6] /= A11[2][2] - - ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] - - a11 += cs_a; - - //extract a33 - ymm1 = _mm256_permute_pd(ymm0, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] - ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11);//1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] - - //(ROw2): FMA operations - ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[3][0-3] -= A11[3][2] * B11[0-3][2] - - ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[7][0-3] -= A11[3][2] * B11[0-3][6] - - //perform mul operation - ymm11 = _mm256_mul_pd(ymm11, ymm1); //B11[0-3][3] /= A11[3][3] - ymm15 = _mm256_mul_pd(ymm15, ymm1); //B11[0-3][7] /= A11[3][3] - - //unpacklow// - ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] - ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] - - ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] - ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] - - //rearrange low elements - ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - - ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] - ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] - - ///unpack high/// - ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] - ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] - - ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[4][1] B11[5][1] B11[4][3] B11[5][3] - ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[6][1] B11[7][1] B11[6][3] B11[7][3] - - //rearrange high elements - ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] - ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] - - _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store B11[0][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store B11[1][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store B11[2][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store B11[3][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store B11[4][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store B11[5][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store B11[6][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store B11[7][0-3] - } - - if(m_remainder) //implementation for reamainder rows(when 'M' is not a multiple of D_MR) - { - a10 = L +i; //pointer to block of A to be used for GEMM - a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM - b01 = B + j*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM - - k_iter = i / D_MR; //number of times GEMM operation to be done(in blocks of 4x4) - - ymm8 = _mm256_setzero_pd(); - ymm9 = _mm256_setzero_pd(); - ymm10 = _mm256_setzero_pd(); - ymm11 = _mm256_setzero_pd(); - ymm12 = _mm256_setzero_pd(); - ymm13 = _mm256_setzero_pd(); - ymm14 = _mm256_setzero_pd(); - ymm15 = _mm256_setzero_pd(); - - ///GEMM code Begins/// - for(k = 0; k< k_iter; k++) //loop for number of GEMM operations - { - ptr_b01_dup = b01; - - ymm16 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] - - b01 += 1; //move to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0] ) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm16 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) - - ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 1)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] - - b01 += 1; //move to next row of B01 - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) - - ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] //A10[1][2] A10[2][2] A10[3][2] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] - - b01 += 1; //move to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][0]*A10[3][2]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][1]*A10[3][2]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][2]*A10[3][2]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] - - b01 += 1; //move to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm8 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm8 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm8 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm8 += (B01[3][0]*A10[0][3] B01[3][4]*A10[1][3] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm8 += (B01[3][1]*A10[0][3] B01[3][5]*A10[1][3] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm8 += (B01[3][2]*A10[0][3] B01[3][6]*A10[1][3] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm8 += (B01[3][3]*A10[0][3] B01[3][7]*A10[1][3] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) - - a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM - b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM - } - - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value - - ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] - ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] - ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] - ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] - - ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] *alpha -= B01[0-3][0] - ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] *alpha -= B01[0-3][1] - ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] *alpha -= B01[0-3][2] - ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] *alpha -= B01[0-3][3] - ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] *alpha -= B01[0-3][4] - ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] *alpha -= B01[0-3][5] - ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] *alpha -= B01[0-3][6] - ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] *alpha -= B01[0-3][7] - - ///implement TRSM/// - - ///unpacklow/// - ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] - ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] - - ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[1][4] B11[1][5] - ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[1][6] B11[1][7] - - //rearrange low elements - ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] - ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] - - ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] - ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] - - ////unpackhigh//// - ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] - ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] - - ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[5][0] B11[5][1] B11[7][0] B11[7][1] - ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[5][2] B11[5][3] B11[7][2] B11[7][3] - - //rearrange high elements - ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] - ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] - - ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] - ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] - - - ymm0 = _mm256_broadcast_sd((double const *)&ones); - - //broadcast diagonal elements of A11 - ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] - ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] - - ymm5 = _mm256_unpacklo_pd(ymm1, ymm2); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] - ymm6 = _mm256_unpacklo_pd(ymm3, ymm4); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] - - ymm5 = _mm256_blend_pd(ymm5, ymm6, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm0 = _mm256_div_pd(ymm0, ymm5); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] - - //extract a00 - ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] - ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] - - //(Row 0): perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - ymm8 = _mm256_mul_pd(ymm8, ymm1); //B11[0-3][0] /= A11[0][0] - ymm12 = _mm256_mul_pd(ymm12, ymm1); //B11[0-3][4] /= A11[0][0] - - //extract a11 - ymm1 = _mm256_permute_pd(ymm0, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] - ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] - - ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] - ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] - ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] - - a11 += cs_a; - - //(Row1): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= B11[0-3][0]*A11[1][0] - ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= B11[0-3][0]*A11[2][0] - ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= B11[0-3][0]*A11[3][0] - - ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= B11[0-3][4]*A11[1][4] - ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= B11[0-3][4]*A11[2][4] - ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= B11[0-3][4]*A11[3][4] - - ymm9 = _mm256_mul_pd(ymm9, ymm1); //B11[0-3][1] /= A11[1][1] - ymm13 = _mm256_mul_pd(ymm13, ymm1); //B11[0-3][5] /= A11[1][1] - - ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] - ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] - - a11 += cs_a; - - //extract a22 - ymm1 = _mm256_permute_pd(ymm0, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] - ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] - - //(ROw2): FMA operations - ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] - ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] - - ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] - ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] - - //perform mul operation - ymm10 = _mm256_mul_pd(ymm10, ymm1); //B11[0-3][2] /=A11[2][2] - ymm14 = _mm256_mul_pd(ymm14, ymm1); //B11[0-3][6] /= A11[2][2] - - ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] - - a11 += cs_a; - - //extract a33 - ymm1 = _mm256_permute_pd(ymm0, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] - ymm1 = _mm256_permute2f128_pd(ymm1, ymm1, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] - - //(ROw2): FMA operations - ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[0-3][3] -= A11[3][2]*B11[0-3][2] - - ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[0-3][7] -= A11[3][2]*B11[0-3][6] - - //perform mul operation - ymm11 = _mm256_mul_pd(ymm11, ymm1); //B11[0-3][3] /= A11[3][3] - ymm15 = _mm256_mul_pd(ymm15, ymm1); //B11[0-3][7] /= A11[3][3] - - //unpacklow// - ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] - ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] - - ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] - ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] - - //rearrange low elements - ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - - ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] - ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] - - ///unpack high/// - ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] - ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] - - ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[0][5] B11[1][5] B11[0][7] B11[1][7] - ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[2][5] B11[3][5] B11[2][7] B11[3][7] - - //rearrange high elements - ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] - ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] - - ymm8 = _mm256_loadu_pd((double const *)(b11 + cs_b * 0)); //load B11[0-3][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b * 1)); //load B11[0-3][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //load B11[0-3][2] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //load B11[0-3][3] - ymm12 = _mm256_loadu_pd((double const *)(b11 + cs_b * 4)); //load B11[0-3][4] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b * 5)); //load B11[0-3][5] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b * 6)); //load B11[0-3][6] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b * 7)); //load B11[0-3][7] - //determine correct values to store - if(m_remainder == 3) - { - ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x08); - ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x08); - ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x08); - ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x08); - ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x08); - ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x08); - ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x08); - ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x08); - } - if(m_remainder == 2) - { - ymm0 = _mm256_permute2f128_pd(ymm0, ymm8, 0x30); - ymm1 = _mm256_permute2f128_pd(ymm1, ymm9, 0x30); - ymm2 = _mm256_permute2f128_pd(ymm2, ymm10, 0x30); - ymm3 = _mm256_permute2f128_pd(ymm3, ymm11, 0x30); - ymm4 = _mm256_permute2f128_pd(ymm4, ymm12, 0x30); - ymm5 = _mm256_permute2f128_pd(ymm5, ymm13, 0x30); - ymm6 = _mm256_permute2f128_pd(ymm6, ymm14, 0x30); - ymm7 = _mm256_permute2f128_pd(ymm7, ymm15, 0x30); - } - if(m_remainder == 1) - { - ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x0E); - ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x0E); - ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x0E); - ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x0E); - ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x0E); - ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x0E); - ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x0E); - ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x0E); - } - - _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) - _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store(B11[0-3][4]) - _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store(B11[0-3][5]) - _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store(B11[0-3][6]) - _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store(B11[0-3][7]) - - } - } - - if((n & 4)) //implementation for remainder columns(when 'N' is a multiple of 4) - { - for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction - { - a10 = L +i; //pointer to block of A to be used for GEMM - a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM - b01 = B + j*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM - - k_iter = i / D_MR; //number of times GEMM to be performed(in block of 4) - ///GEMM for previously calculated values /// - - //load 4x4 block from b11 - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b*3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_b01_dup = b01; - ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a*2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] - - b01 += 1; //move to next row of B - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[1][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[2][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[3][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) - - - a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM - b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM - - } - - ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 - ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B01[0-3][1] *alpha -= ymm5 - ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B01[0-3][2] *alpha -= ymm6 - ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B01[0-3][3] *alpha -= ymm7 - - ///implement TRSM/// - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] - ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] - - //2nd col - a11 += cs_a; - ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] - - //3rd col - a11 += cs_a; - ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] - ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] - - //4th col - a11 += cs_a; - ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] - //compute reciprocals of L(i,i) and broadcast in registers - ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[0][0] A11[2][2] A11[2][2] - ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[1][1] A11[1][1] A11[3][3] A11[3][3] - - ymm14 = _mm256_broadcast_sd((double const *)&ones); - - ymm4 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm14 = _mm256_div_pd(ymm14, ymm4); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] - - ////unpacklow//// - ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] - ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] - - //rearrange low elements - ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] - ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] -/* - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); -*/ - ////unpackhigh//// - ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] - ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] - - //rearrange high elements - ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] - ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] -/* - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); -*/ - //extract a00 - ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - ymm4 = _mm256_mul_pd(ymm4, ymm15); //B11[0][0-3] /= A11[0][0] - - //extract diag a11 from a - ymm15 = _mm256_permute_pd(ymm14, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]*B11[0][0-3] - ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]*B11[0][0-3] - ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]*B11[0][0-3] - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - ymm8 = _mm256_mul_pd(ymm8, ymm15); //B11[1][0-3] /= A11[1][1] - - - //extract diag a22 from a - ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]*B11[1][0-3] - ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]*B11[1][0-3] - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - ymm11 = _mm256_mul_pd(ymm11, ymm15); //B11[2][0-3] /= A11[2][2] - - //extract diag a33 from a - ymm15 = _mm256_permute_pd(ymm14, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]*B11[2][0-3] - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - ymm13 = _mm256_mul_pd(ymm13, ymm15); //B11[3][0-3] /= A11[3][3] - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] - ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] - - //rearrange low elements - ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - - ////unpackhigh//// - ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] - - ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] - - //rearrange high elements - ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b*2), ymm2); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[0-3][3]) - - } - if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) - { - a10 = L +i; //pointer to block of A to be used for GEMM - a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM - b01 = B + j*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha - - k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) - - ///GEMM for previously calculated values /// - - //load 4x4 block from b11 - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - for(k = 0; k < k_iter; k++) //looop for number of GEMM operations - { - ptr_b01_dup = b01; - - ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) - - a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM - b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM - - } - - ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 - ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 - ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 - ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 - - ///implement TRSM/// - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] - ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] - - //2nd col - a11 += cs_a; - ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] - - //3rd col - a11 += cs_a; - ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] - ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] - - //4th col - a11 += cs_a; - ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] - //compute reciprocals of L(i,i) and broadcast in registers - ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] - ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] - - ymm14 = _mm256_broadcast_sd((double const *)&ones); - - ymm4 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm14 = _mm256_div_pd(ymm14, ymm4); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] - - ////unpacklow//// - ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] - ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] - - //rearrange low elements - ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] - ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] -/* - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); -*/ - ////unpackhigh//// - ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] - ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] - - //rearrange high elements - ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] - ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] -/* - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); -*/ - //extract a00 - ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00);//1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - ymm4 = _mm256_mul_pd(ymm4, ymm15); //B11[0][0-3] /= A11[0][0] - - //extract diag a11 from a - ymm15 = _mm256_permute_pd(ymm14, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[3][3] 1/A11[3][3] - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[][] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]* B11[0][0-3] - ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]* B11[0][0-3] - ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]* B11[0][0-3] - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - ymm8 = _mm256_mul_pd(ymm8, ymm15); //B11[1][0-3] /= A11[1][1] - - - //extract diag a22 from a - ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]* B11[1][0-3] - ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]* B11[1][0-3] - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - ymm11 = _mm256_mul_pd(ymm11, ymm15); //B11[2][0-3] /= A11[2][2] - - //extract diag a33 from a - ymm15 = _mm256_permute_pd(ymm14, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]* B11[2][0-3] - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - ymm13 = _mm256_mul_pd(ymm13, ymm15); //B11[3][0-3] /= A11[3][3] - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] - ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] - - //rearrange low elements - ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - - ////unpackhigh//// - ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] - - ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] - - //rearrange high elements - ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - //load 4x4 block from b11 - ymm4 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][2] B11[3][3] - - //determine correct values to store - - if(m_remainder == 3) - { - ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x08); - ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x08); - ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x08); - ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x08); - } - if(m_remainder == 2) - { - ymm0 = _mm256_permute2f128_pd(ymm0, ymm4,0x30); - ymm1 = _mm256_permute2f128_pd(ymm1, ymm5,0x30); - ymm2 = _mm256_permute2f128_pd(ymm2, ymm6,0x30); - ymm3 = _mm256_permute2f128_pd(ymm3, ymm7,0x30); - } - if(m_remainder == 1) - { - ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x0E); - ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x0E); - ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x0E); - ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x0E); - } - - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) - - } - - n_remainder -= 4; - j += 4; - - } - - if(n_remainder) //implementation fo remaining columns(when 'N' is not a multiple of D_NR) - { - for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction - { - a10 = L +i; //pointer to block of A to be used for GEMM - a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM - b01 = B + j*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM - - k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha Value - - ///GEMM for previously calculated values /// - - //load 4x4 block from b11 - if(n_remainder == 3) - { - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_broadcast_sd((double const *)&ones); - } - if(n_remainder == 2) - { - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_broadcast_sd((double const *)&ones); - ymm3 = _mm256_broadcast_sd((double const *)&ones); - } - if(n_remainder == 1) - { - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_broadcast_sd((double const *)&ones); - ymm2 = _mm256_broadcast_sd((double const *)&ones); - ymm3 = _mm256_broadcast_sd((double const*)&ones); - } - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - for(k = 0; k < k_iter; k++) - { - ptr_b01_dup = b01; - ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) - - a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM - b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM - } - - ///GEMM code ends/// - - ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 - ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 - ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 - ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 - - ///implement TRSM/// - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] - ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] - - //2nd col - a11 += cs_a; - ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] - - //3rd col - a11 += cs_a; - ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] - ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] - - //4th col - a11 += cs_a; - ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] - //compute reciprocals of L(i,i) and broadcast in registers - ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[0][0] A11[1][1] A11[1][1] - ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[2][2] A11[3][3] A11[3][3] - - ymm14 = _mm256_broadcast_sd((double const *)&ones); - - ymm4 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm14 = _mm256_div_pd(ymm14, ymm4); //1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] - - ////unpacklow//// - ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] - ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] - - //rearrange low elements - ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] - ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] -/* - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); -*/ - ////unpackhigh//// - ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] - ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] - - //rearrange high elements - ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] - ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] -/* - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); -*/ - //extract a00 - ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0] - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - ymm4 = _mm256_mul_pd(ymm4, ymm15); //B11[0][0-3] /= A11[0][0] - - //extract diag a11 from a - ymm15 = _mm256_permute_pd(ymm14, 0x03); //1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2] - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1] - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0] * B11[0][0-3] - ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0] * B11[0][0-3] - ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0] * B11[0][0-3] - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - ymm8 = _mm256_mul_pd(ymm8, ymm15); //B11[1][0-3] /= A11[1][1] - - - //extract diag a22 from a - ymm15 = _mm256_permute_pd(ymm14, 0x00); //1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2] - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2] - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1] * B11[1][0-3] - ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1] * B11[1][0-3] - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - ymm11 = _mm256_mul_pd(ymm11, ymm15); //B11[2][0-3] /= A11[2][2] - - //extract diag a33 from a - ymm15 = _mm256_permute_pd(ymm14, 0x0C); //1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3] - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2] * B11[2][0-3] - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - ymm13 = _mm256_mul_pd(ymm13, ymm15); //B11[3][0-3] /= A11[3][3] - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] - ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] - - //rearrange low elements - ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - - ////unpackhigh//// - ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] - - ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] - - //rearrange high elements - ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) - - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) - - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - } - - } - if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) - { - a10 = L +i; //pointer to block of A to be used for GEMM - a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM - b01 = B + j*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM - - - k_iter = i / D_MR; //number of times GEMM operations to be performed - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha value - - ///GEMM for previously calculated values /// - - - //load 4x4 block from b11 - if(n_remainder == 3) - { - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_broadcast_sd((double const *)&ones); - } - if(n_remainder == 2) - { - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_broadcast_sd((double const *)&ones); - ymm3 = _mm256_broadcast_sd((double const *)&ones); - } - if(n_remainder == 1) - { - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_broadcast_sd((double const *)&ones); - ymm2 = _mm256_broadcast_sd((double const *)&ones); - ymm3 = _mm256_broadcast_sd((double const *)&ones); - } - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_b01_dup = b01; - ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[0][3] - - b01 += 1; //move to next row of B - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[1][3] - - b01 += 1; //move to next row of B - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[2][3] - - b01 += 1; //move to next row of B - - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[3][3] - - b01 += 1; //move to next row of B - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) - - a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM - b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM - - } - - ymm8 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] * alpha -= ymm4 - ymm9 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] * alpha -= ymm5 - ymm10 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] * alpha -= ymm6 - ymm11 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] * alpha -= ymm7 - - ///implement TRSM/// - //determine correct values to store - if(m_remainder == 3) - { - ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x08); - ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x08); - ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x08); - ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x08); - - } - if(m_remainder == 2) - { - ymm0 = _mm256_permute2f128_pd(ymm8, ymm0, 0x30); - ymm1 = _mm256_permute2f128_pd(ymm9, ymm1, 0x30); - ymm2 = _mm256_permute2f128_pd(ymm10, ymm2, 0x30); - ymm3 = _mm256_permute2f128_pd(ymm11, ymm3, 0x30); - - } - if(m_remainder == 1) - { - ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x0E); - ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x0E); - ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x0E); - ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x0E); - } - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - } - - ///scalar code for trsm without alpha/// - dtrsm_small_AlXB(a11, b11, m_remainder, n_remainder, cs_a, cs_b); - } - } - return BLIS_SUCCESS; -} - -/* TRSM for the case AX = alpha * B, Double precision - * A is lower-triangular, no-transpose, unit diagonal - * dimensions A: mxm X: mxn B: mxn - - b01---> - * ***************** - ** * * * * * - * * * * * * * - * * *b01* * * * - * * * * * * * -a10 ****** b11 ***************** - | * * * | * * * * * - | * * * | * * * * * - | *a10*a11* | *b11* * * * - v * * * v * * * * * - *********** ***************** - * * * * * * * * * - * * * * * * * * * - * * * * * * * * * - * * * * * * * * * - **************** ***************** - a11---> -*/ - -static err_t bli_dtrsm_small_AlXB_unitDiag( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - - dim_t D_MR = 4; //size of block along 'M' dimpension - dim_t D_NR = 8; //size of block along 'N' dimension - - dim_t m = bli_obj_length(b); // number of rows of matrix B - dim_t n = bli_obj_width(b); // number of columns of matrix B - - if(max(m,n) > 90) - return BLIS_NOT_YET_IMPLEMENTED; - - dim_t m_remainder = m % D_MR; //number of remainder rows - dim_t n_remainder = n % D_NR; //number of remainder columns - - dim_t cs_a = bli_obj_col_stride(a); // column stride of A - dim_t cs_b = bli_obj_col_stride(b); // column stride of B - - dim_t i, j, k; //loop variables - dim_t k_iter; //number of times GEMM to be performed - - double AlphaVal = *(double *)AlphaObj->buffer; //value of alpha - double *L = a->buffer; //pointer to matrix A - double *B = b->buffer; //pointer to matrix B - - double *a10, *a11, *b01, *b11; //pointers that point to blocks for GEMM and TRSM - double *ptr_b01_dup; - - double ones = 1.0; - - //scratch registers - __m256d ymm0, ymm1, ymm2, ymm3; - __m256d ymm4, ymm5, ymm6, ymm7; - __m256d ymm8, ymm9, ymm10, ymm11; - __m256d ymm12, ymm13, ymm14, ymm15; - __m256d ymm16; - - - - for(j = 0; j+D_NR-1 < n; j += D_NR) //loop along 'N' dimension - { - for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' dimension - { - a10 = L +i; //pointer to block of A to be used for GEMM - a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM - b01 = B + j*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM - - k_iter = i / D_MR; //number of times GEMM to be performed(in blocks of 4x4) - - ymm8 = _mm256_setzero_pd(); - ymm9 = _mm256_setzero_pd(); - ymm10 = _mm256_setzero_pd(); - ymm11 = _mm256_setzero_pd(); - ymm12 = _mm256_setzero_pd(); - ymm13 = _mm256_setzero_pd(); - ymm14 = _mm256_setzero_pd(); - ymm15 = _mm256_setzero_pd(); - - ///GEMM code begins/// - - for(k = 0; k< k_iter; k++) //loop for number of GEMM operations - { - ptr_b01_dup = b01; - - ymm16 = _mm256_loadu_pd((double const *)(a10));//A10[0][0] A10[1][0] A10[2][0] A10[3][0] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] - - b01 += 1; //mobe to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) - - ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a));//A10[0][1] A10[1][1] A10[2][1] A10[3][1] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] - - b01 += 1; //mobe to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) - - ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2));//A10[0][2] A10[1][2] A10[2][2] A10[3][2] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] - - b01 += 1; //mobe to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][4]*A10[3][2]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][5]*A10[3][2]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][6]*A10[3][2]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][7]*A10[3][2]) - - ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3));//A10[0][3] A10[1][3] A10[2][3] A10[3][3] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] - - b01 += 1; //mobe to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[3][0] B01[3][0]*A10[2][3] B01[3][0]*A10[3][0]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[3][1]*A10[0][3] B01[3][1]*A10[3][0] B01[3][1]*A10[2][3] B01[3][1]*A10[3][0]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[3][2]*A10[0][3] B01[3][2]*A10[3][0] B01[3][2]*A10[2][3] B01[3][2]*A10[3][0]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[3][3]*A10[0][3] B01[3][3]*A10[3][0] B01[3][3]*A10[2][3] B01[3][3]*A10[3][0]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[3][4]*A10[0][3] B01[3][4]*A10[3][0] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[3][5]*A10[0][3] B01[3][5]*A10[3][0] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[3][6]*A10[0][3] B01[3][6]*A10[3][0] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[3][7]*A10[0][3] B01[3][7]*A10[3][0] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) - - a10 += D_MR * cs_a; //pointer math to calculate next block of A for GEMM - b01 = ptr_b01_dup + D_MR; //pointer math to calculate next block of B for GEMM - } - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha - - ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] - ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] - ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] - ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] - - ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] * alpha -= B01[0-3][0] - ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] * alpha -= B01[0-3][1] - ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] * alpha -= B01[0-3][2] - ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] * alpha -= B01[0-3][3] - ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] * alpha -= B01[0-3][4] - ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] * alpha -= B01[0-3][5] - ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] * alpha -= B01[0-3][6] - ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] * alpha -= B01[0-3][7] - - ///implement TRSM/// - - ///transpose of B11// - ///unpacklow/// - ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] - ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] - - ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[2][4] B11[2][5] - ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[2][6] B11[2][7] - - //rearrange low elements - ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] - ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] - - ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] - ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] - - ////unpackhigh//// - ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] - ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] - - ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[1][4] B11[1][5] B11[3][4] B11[3][5] - ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[1][6] B11[1][7] B11[3][6] B11[3][7] - - //rearrange high elements - ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] - ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] - - ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] - ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] - - //broadcast diagonal elements of A11 - ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] - ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] - - ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] - ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] - ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] - - a11 += cs_a; - - //(Row1): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= A11[1][0] * B11[0-3][0] - ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= A11[2][0] * B11[0-3][0] - ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= A11[3][0] * B11[0-3][0] - - ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= A11[1][0] * B11[0-3][4] - ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= A11[2][0] * B11[0-3][4] - ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= A11[3][0] * B11[0-3][4] - - ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] - ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] - - a11 += cs_a; - - //(ROw2): FMA operations - ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] - ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] - - ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] - ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] - - ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] - - a11 += cs_a; - - //(ROw2): FMA operations - ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[3][0-3] -= A11[3][2] * B11[0-3][2] - - ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[7][0-3] -= A11[3][2] * B11[0-3][6] - - //unpacklow// - ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] - ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] - - ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] - ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] - - //rearrange low elements - ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - - ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] - ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] - - ///unpack high/// - ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] - ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] - - ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[4][1] B11[5][1] B11[4][3] B11[5][3] - ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[6][1] B11[7][1] B11[6][3] B11[7][3] - - //rearrange high elements - ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] - ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] - - _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store B11[0][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store B11[1][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store B11[2][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store B11[3][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store B11[4][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store B11[5][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store B11[6][0-3] - _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store B11[7][0-3] - } - - if(m_remainder) //implementation for reamainder rows(when 'M' is not a multiple of D_MR) - { - a10 = L +i; //pointer to block of A to be used for GEMM - a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM - b01 = B + j*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM - - k_iter = i / D_MR; //number of times GEMM operation to be done(in blocks of 4x4) - - ymm8 = _mm256_setzero_pd(); - ymm9 = _mm256_setzero_pd(); - ymm10 = _mm256_setzero_pd(); - ymm11 = _mm256_setzero_pd(); - ymm12 = _mm256_setzero_pd(); - ymm13 = _mm256_setzero_pd(); - ymm14 = _mm256_setzero_pd(); - ymm15 = _mm256_setzero_pd(); - - ///GEMM code Begins/// - for(k = 0; k< k_iter; k++) //loop for number of GEMM operations - { - ptr_b01_dup = b01; - - ymm16 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[0][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[0][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[0][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[0][7] - - b01 += 1; //move to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0] ) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[0][4]*A10[0][0] B01[0][4]*A10[1][0] B01[0][4]*A10[2][0] B01[0][4]*A10[3][0]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[0][5]*A10[0][0] B01[0][5]*A10[1][0] B01[0][5]*A10[2][0] B01[0][5]*A10[3][0]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[0][6]*A10[0][0] B01[0][6]*A10[1][0] B01[0][6]*A10[2][0] B01[0][6]*A10[3][0]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm16 += (B01[0][7]*A10[0][0] B01[0][7]*A10[1][0] B01[0][7]*A10[2][0] B01[0][7]*A10[3][0]) - - ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 1)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[1][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[1][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[1][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[1][7] - - b01 += 1; //move to next row of B01 - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[1][4]*A10[0][1] B01[1][4]*A10[1][1] B01[1][4]*A10[2][1] B01[1][4]*A10[3][1]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[1][5]*A10[0][1] B01[1][5]*A10[1][1] B01[1][5]*A10[2][1] B01[1][5]*A10[3][1]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[1][6]*A10[0][1] B01[1][6]*A10[1][1] B01[1][6]*A10[2][1] B01[1][6]*A10[3][1]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[1][7]*A10[0][1] B01[1][7]*A10[1][1] B01[1][7]*A10[2][1] B01[1][7]*A10[3][1]) - - ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] //A10[1][2] A10[2][2] A10[3][2] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[2][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[2][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[2][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[2][7] - - b01 += 1; //move to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm9 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm10 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm11 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm12 += (B01[2][4]*A10[0][2] B01[2][4]*A10[1][2] B01[2][4]*A10[2][2] B01[2][0]*A10[3][2]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm13 += (B01[2][5]*A10[0][2] B01[2][5]*A10[1][2] B01[2][5]*A10[2][2] B01[2][1]*A10[3][2]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm14 += (B01[2][6]*A10[0][2] B01[2][6]*A10[1][2] B01[2][6]*A10[2][2] B01[2][2]*A10[3][2]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm15 += (B01[2][7]*A10[0][2] B01[2][7]*A10[1][2] B01[2][7]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm16 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] - - ymm4 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] - ymm5 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] - ymm6 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] - ymm7 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] - - ymm0 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 4)); //B01[3][4] - ymm1 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 5)); //B01[3][5] - ymm2 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 6)); //B01[3][6] - ymm3 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 7)); //B01[3][7] - - b01 += 1; //move to next row of B - - ymm8 = _mm256_fmadd_pd(ymm4, ymm16, ymm8); //ymm8 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) - ymm9 = _mm256_fmadd_pd(ymm5, ymm16, ymm9); //ymm8 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) - ymm10 = _mm256_fmadd_pd(ymm6, ymm16, ymm10); //ymm8 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) - ymm11 = _mm256_fmadd_pd(ymm7, ymm16, ymm11); //ymm8 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) - - ymm12 = _mm256_fmadd_pd(ymm0, ymm16, ymm12); //ymm8 += (B01[3][0]*A10[0][3] B01[3][4]*A10[1][3] B01[3][4]*A10[2][3] B01[3][4]*A10[3][3]) - ymm13 = _mm256_fmadd_pd(ymm1, ymm16, ymm13); //ymm8 += (B01[3][1]*A10[0][3] B01[3][5]*A10[1][3] B01[3][5]*A10[2][3] B01[3][5]*A10[3][3]) - ymm14 = _mm256_fmadd_pd(ymm2, ymm16, ymm14); //ymm8 += (B01[3][2]*A10[0][3] B01[3][6]*A10[1][3] B01[3][6]*A10[2][3] B01[3][6]*A10[3][3]) - ymm15 = _mm256_fmadd_pd(ymm3, ymm16, ymm15); //ymm8 += (B01[3][3]*A10[0][3] B01[3][7]*A10[1][3] B01[3][7]*A10[2][3] B01[3][7]*A10[3][3]) - - a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM - b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM - } - - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value - - ymm0 = _mm256_loadu_pd((double const *)(b11 + cs_b *0)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b *1)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b *2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b *3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm4 = _mm256_loadu_pd((double const *)(b11 + cs_b *4)); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] - ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b *5)); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] - ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b *6)); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] - ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b *7)); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] - - ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm8); //B11[0-3][0] *alpha -= B01[0-3][0] - ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm9); //B11[0-3][1] *alpha -= B01[0-3][1] - ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm10); //B11[0-3][2] *alpha -= B01[0-3][2] - ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm11); //B11[0-3][3] *alpha -= B01[0-3][3] - ymm4 = _mm256_fmsub_pd(ymm4, ymm16, ymm12); //B11[0-3][4] *alpha -= B01[0-3][4] - ymm5 = _mm256_fmsub_pd(ymm5, ymm16, ymm13); //B11[0-3][5] *alpha -= B01[0-3][5] - ymm6 = _mm256_fmsub_pd(ymm6, ymm16, ymm14); //B11[0-3][6] *alpha -= B01[0-3][6] - ymm7 = _mm256_fmsub_pd(ymm7, ymm16, ymm15); //B11[0-3][7] *alpha -= B01[0-3][7] - - ///implement TRSM/// - - ///unpacklow/// - ymm9 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] - ymm11 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] - - ymm13 = _mm256_unpacklo_pd(ymm4, ymm5); //B11[0][4] B11[0][5] B11[1][4] B11[1][5] - ymm15 = _mm256_unpacklo_pd(ymm6, ymm7); //B11[0][6] B11[0][7] B11[1][6] B11[1][7] - - //rearrange low elements - ymm8 = _mm256_permute2f128_pd(ymm9,ymm11,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] - ymm10 = _mm256_permute2f128_pd(ymm9,ymm11,0x31); //B11[2][0] B11[2][1] B11[2][2] B11[2][3] - - ymm12 = _mm256_permute2f128_pd(ymm13,ymm15,0x20); //B11[4][0] B11[4][1] B11[4][2] B11[4][3] - ymm14 = _mm256_permute2f128_pd(ymm13,ymm15,0x31); //B11[6][0] B11[6][1] B11[6][2] B11[6][3] - - ////unpackhigh//// - ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] - ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] - - ymm4 = _mm256_unpackhi_pd(ymm4, ymm5); //B11[5][0] B11[5][1] B11[7][0] B11[7][1] - ymm5 = _mm256_unpackhi_pd(ymm6, ymm7); //B11[5][2] B11[5][3] B11[7][2] B11[7][3] - - //rearrange high elements - ymm9 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] - ymm11 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] - - ymm13 = _mm256_permute2f128_pd(ymm4,ymm5,0x20); //B11[5][0] B11[5][1] B11[5][2] B11[5][3] - ymm15 = _mm256_permute2f128_pd(ymm4,ymm5,0x31); //B11[7][0] B11[7][1] B11[7][2] B11[7][3] - - //broadcast diagonal elements of A11 - ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_b +1)); //A11[1][1] - ymm3 = _mm256_broadcast_sd((double const *)(a11+cs_b*2 + 2)); //A11[2][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+cs_b*3 + 3)); //A11[3][3] - - ymm2 = _mm256_broadcast_sd((double const *)(a11 +1)); //A11[1][0] - ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][0] - ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][0] - - a11 += cs_a; - - //(Row1): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm2, ymm8, ymm9); //B11[1][0-3] -= B11[0-3][0]*A11[1][0] - ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[2][0-3] -= B11[0-3][0]*A11[2][0] - ymm11 = _mm256_fnmadd_pd(ymm4, ymm8, ymm11); //B11[3][0-3] -= B11[0-3][0]*A11[3][0] - - ymm13 = _mm256_fnmadd_pd(ymm2, ymm12, ymm13); //B11[5][0-3] -= B11[0-3][4]*A11[1][4] - ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[6][0-3] -= B11[0-3][4]*A11[2][4] - ymm15 = _mm256_fnmadd_pd(ymm4, ymm12, ymm15); //B11[7][0-3] -= B11[0-3][4]*A11[3][4] - - ymm3 = _mm256_broadcast_sd((double const *)(a11 +2)); //A11[2][1] - ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][1] - - a11 += cs_a; - - //(ROw2): FMA operations - ymm10 = _mm256_fnmadd_pd(ymm3, ymm9, ymm10); //B11[2][0-3] -= A11[2][1] * B11[0-3][1] - ymm11 = _mm256_fnmadd_pd(ymm4, ymm9, ymm11); //B11[3][0-3] -= A11[3][1] * B11[0-3][1] - - ymm14 = _mm256_fnmadd_pd(ymm3, ymm13, ymm14); //B11[6][0-3] -= A11[2][1] * B11[0-3][5] - ymm15 = _mm256_fnmadd_pd(ymm4, ymm13, ymm15); //B11[7][0-3] -= A11[3][1] * B11[0-3][5] - - ymm4 = _mm256_broadcast_sd((double const *)(a11 +3)); //A11[3][2] - - a11 += cs_a; - - //(ROw2): FMA operations - ymm11 = _mm256_fnmadd_pd(ymm4, ymm10, ymm11); //B11[0-3][3] -= A11[3][2]*B11[0-3][2] - - ymm15 = _mm256_fnmadd_pd(ymm4, ymm14, ymm15); //B11[0-3][7] -= A11[3][2]*B11[0-3][6] - - //unpacklow// - ymm1 = _mm256_unpacklo_pd(ymm8, ymm9); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] - ymm3 = _mm256_unpacklo_pd(ymm10, ymm11); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] - - ymm5 = _mm256_unpacklo_pd(ymm12, ymm13); //B11[4][0] B11[5][0] B11[4][2] B11[5][2] - ymm7 = _mm256_unpacklo_pd(ymm14, ymm15); //B11[6][0] B11[7][0] B11[6][2] B11[7][2] - - //rearrange low elements - ymm0 = _mm256_permute2f128_pd(ymm1, ymm3, 0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_permute2f128_pd(ymm1, ymm3, 0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - - ymm4 = _mm256_permute2f128_pd(ymm5, ymm7, 0x20); //B11[0][4] B11[1][4] B11[2][4] B11[3][4] - ymm6 = _mm256_permute2f128_pd(ymm5, ymm7, 0x31); //B11[0][6] B11[1][6] B11[2][6] B11[3][6] - - ///unpack high/// - ymm8 = _mm256_unpackhi_pd(ymm8, ymm9); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] - ymm9 = _mm256_unpackhi_pd(ymm10, ymm11); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] - - ymm12 = _mm256_unpackhi_pd(ymm12, ymm13); //B11[0][5] B11[1][5] B11[0][7] B11[1][7] - ymm13 = _mm256_unpackhi_pd(ymm14, ymm15); //B11[2][5] B11[3][5] B11[2][7] B11[3][7] - - //rearrange high elements - ymm1 = _mm256_permute2f128_pd(ymm8, ymm9, 0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_permute2f128_pd(ymm8, ymm9, 0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - ymm5 = _mm256_permute2f128_pd(ymm12, ymm13, 0x20); //B11[0][5] B11[1][5] B11[2][5] B11[3][5] - ymm7 = _mm256_permute2f128_pd(ymm12, ymm13, 0x31); //B11[0][7] B11[1][7] B11[2][7] B11[3][7] - - ymm8 = _mm256_loadu_pd((double const *)(b11 + cs_b * 0)); //load B11[0-3][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b * 1)); //load B11[0-3][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //load B11[0-3][2] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //load B11[0-3][3] - ymm12 = _mm256_loadu_pd((double const *)(b11 + cs_b * 4)); //load B11[0-3][4] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b * 5)); //load B11[0-3][5] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b * 6)); //load B11[0-3][6] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b * 7)); //load B11[0-3][7] - //determine correct values to store - if(m_remainder == 3) - { - ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x08); - ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x08); - ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x08); - ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x08); - ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x08); - ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x08); - ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x08); - ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x08); - } - if(m_remainder == 2) - { - ymm0 = _mm256_permute2f128_pd(ymm0, ymm8, 0x30); - ymm1 = _mm256_permute2f128_pd(ymm1, ymm9, 0x30); - ymm2 = _mm256_permute2f128_pd(ymm2, ymm10, 0x30); - ymm3 = _mm256_permute2f128_pd(ymm3, ymm11, 0x30); - ymm4 = _mm256_permute2f128_pd(ymm4, ymm12, 0x30); - ymm5 = _mm256_permute2f128_pd(ymm5, ymm13, 0x30); - ymm6 = _mm256_permute2f128_pd(ymm6, ymm14, 0x30); - ymm7 = _mm256_permute2f128_pd(ymm7, ymm15, 0x30); - } - if(m_remainder == 1) - { - ymm0 = _mm256_blend_pd(ymm0, ymm8, 0x0E); - ymm1 = _mm256_blend_pd(ymm1, ymm9, 0x0E); - ymm2 = _mm256_blend_pd(ymm2, ymm10, 0x0E); - ymm3 = _mm256_blend_pd(ymm3, ymm11, 0x0E); - ymm4 = _mm256_blend_pd(ymm4, ymm12, 0x0E); - ymm5 = _mm256_blend_pd(ymm5, ymm13, 0x0E); - ymm6 = _mm256_blend_pd(ymm6, ymm14, 0x0E); - ymm7 = _mm256_blend_pd(ymm7, ymm15, 0x0E); - } - - _mm256_storeu_pd((double *)(b11 + cs_b * 0), ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b * 1), ymm1); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) - _mm256_storeu_pd((double *)(b11 + cs_b * 4), ymm4); //store(B11[0-3][4]) - _mm256_storeu_pd((double *)(b11 + cs_b * 5), ymm5); //store(B11[0-3][5]) - _mm256_storeu_pd((double *)(b11 + cs_b * 6), ymm6); //store(B11[0-3][6]) - _mm256_storeu_pd((double *)(b11 + cs_b * 7), ymm7); //store(B11[0-3][7]) - - } - } - - if((n & 4)) //implementation for remainder columns(when 'N' is a multiple of 4) - { - for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction - { - a10 = L +i; //pointer to block of A to be used for GEMM - a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM - b01 = B + j*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM - - k_iter = i / D_MR; //number of times GEMM to be performed(in block of 4) - ///GEMM for previously calculated values /// - - //load 4x4 block from b11 - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b*3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_b01_dup = b01; - ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a*2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] - - b01 += 1; //move to next row of B - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[1][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[2][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[3][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) - - - a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM - b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM - - } - - ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 - ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B01[0-3][1] *alpha -= ymm5 - ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B01[0-3][2] *alpha -= ymm6 - ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B01[0-3][3] *alpha -= ymm7 - - ///implement TRSM/// - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] - ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] - - //2nd col - a11 += cs_a; - ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] - - //3rd col - a11 += cs_a; - ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] - ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] - - //4th col - a11 += cs_a; - ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] - - ////unpacklow//// - ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] - ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] - - //rearrange low elements - ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] - ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] - - ////unpackhigh//// - ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] - ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] - - //rearrange high elements - ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] - ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]*B11[0][0-3] - ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]*B11[0][0-3] - ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]*B11[0][0-3] - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]*B11[1][0-3] - ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]*B11[1][0-3] - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]*B11[2][0-3] - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] - ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] - - //rearrange low elements - ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - - ////unpackhigh//// - ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] - - ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] - - //rearrange high elements - ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b*2), ymm2); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[0-3][3]) - - } - if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) - { - a10 = L +i; //pointer to block of A to be used for GEMM - a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM - b01 = B + j*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha - - k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) - - ///GEMM for previously calculated values /// - - //load 4x4 block from b11 - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - for(k = 0; k < k_iter; k++) //looop for number of GEMM operations - { - ptr_b01_dup = b01; - - ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) - - a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM - b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM - - } - - ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 - ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 - ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 - ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 - - ///implement TRSM/// - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] - ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] - - //2nd col - a11 += cs_a; - ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] - - //3rd col - a11 += cs_a; - ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] - ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] - - //4th col - a11 += cs_a; - ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] - - ////unpacklow//// - ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] - ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] - - //rearrange low elements - ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] - ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] - - ////unpackhigh//// - ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] - ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] - - //rearrange high elements - ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] - ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0]* B11[0][0-3] - ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0]* B11[0][0-3] - ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0]* B11[0][0-3] - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1]* B11[1][0-3] - ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1]* B11[1][0-3] - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2]* B11[2][0-3] - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] - ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] - - //rearrange low elements - ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - - ////unpackhigh//// - ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] - - ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] - - //rearrange high elements - ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - //load 4x4 block from b11 - ymm4 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm5 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm6 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm7 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][3] B11[1][3] B11[2][2] B11[3][3] - - //determine correct values to store - - if(m_remainder == 3) - { - ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x08); - ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x08); - ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x08); - ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x08); - } - if(m_remainder == 2) - { - ymm0 = _mm256_permute2f128_pd(ymm0, ymm4,0x30); - ymm1 = _mm256_permute2f128_pd(ymm1, ymm5,0x30); - ymm2 = _mm256_permute2f128_pd(ymm2, ymm6,0x30); - ymm3 = _mm256_permute2f128_pd(ymm3, ymm7,0x30); - } - if(m_remainder == 1) - { - ymm0 = _mm256_blend_pd(ymm0, ymm4, 0x0E); - ymm1 = _mm256_blend_pd(ymm1, ymm5, 0x0E); - ymm2 = _mm256_blend_pd(ymm2, ymm6, 0x0E); - ymm3 = _mm256_blend_pd(ymm3, ymm7, 0x0E); - } - - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[0-3][3]) - - } - - n_remainder -= 4; - j += 4; - - } - - if(n_remainder) //implementation fo remaining columns(when 'N' is not a multiple of D_NR) - { - for(i = 0;i+D_MR-1 < m; i += D_MR) //loop along 'M' direction - { - a10 = L +i; //pointer to block of A to be used for GEMM - a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM - b01 = B + j*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM - - k_iter = i / D_MR; //number of GEMM operations to be performed(in blocks of 4x4) - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha Value - - ///GEMM for previously calculated values /// - - //load 4x4 block from b11 - if(n_remainder == 3) - { - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_broadcast_sd((double const *)&ones); - } - if(n_remainder == 2) - { - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_broadcast_sd((double const *)&ones); - ymm3 = _mm256_broadcast_sd((double const *)&ones); - } - if(n_remainder == 1) - { - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_broadcast_sd((double const *)&ones); - ymm2 = _mm256_broadcast_sd((double const *)&ones); - ymm3 = _mm256_broadcast_sd((double const*)&ones); - } - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - for(k = 0; k < k_iter; k++) - { - ptr_b01_dup = b01; - ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[0][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[1][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[2][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B01[3][3] - - b01 += 1; - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) - - a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM - b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM - } - - ///GEMM code ends/// - - ymm0 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] *alpha -= ymm4 - ymm1 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] *alpha -= ymm5 - ymm2 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] *alpha -= ymm6 - ymm3 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] *alpha -= ymm7 - - ///implement TRSM/// - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][0] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][0] - ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][0] - - //2nd col - a11 += cs_a; - ymm8 = _mm256_broadcast_sd((double const *)(a11 + 1)); //A11[1][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][1] - - //3rd col - a11 += cs_a; - ymm11 = _mm256_broadcast_sd((double const *)(a11 + 2)); //A11[2][2] - ymm12 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][2] - - //4th col - a11 += cs_a; - ymm13 = _mm256_broadcast_sd((double const *)(a11 + 3)); //A11[3][3] - - ////unpacklow//// - ymm8 = _mm256_unpacklo_pd(ymm0, ymm1); //B11[0][0] B11[0][1] B11[2][0] B11[2][1] - ymm13 = _mm256_unpacklo_pd(ymm2, ymm3); //B11[0][2] B11[0][3] B11[2][2] B11[2][3] - - //rearrange low elements - ymm4 = _mm256_permute2f128_pd(ymm8,ymm13,0x20); //B11[0][0] B11[0][1] B11[0][2] B11[0][3] - ymm11 = _mm256_permute2f128_pd(ymm8,ymm13,0x31);//B11[2][0] B11[2][1] B11[2][2] B11[2][3] - - ////unpackhigh//// - ymm0 = _mm256_unpackhi_pd(ymm0, ymm1); //B11[1][0] B11[1][1] B11[3][0] B11[3][1] - ymm1 = _mm256_unpackhi_pd(ymm2, ymm3); //B11[1][2] B11[1][3] B11[3][2] B11[3][3] - - //rearrange high elements - ymm8 = _mm256_permute2f128_pd(ymm0,ymm1,0x20); //B11[1][0] B11[1][1] B11[1][2] B11[1][3] - ymm13 = _mm256_permute2f128_pd(ymm0,ymm1,0x31); //B11[3][0] B11[3][1] B11[3][2] B11[3][3] - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - ymm8 = _mm256_fnmadd_pd(ymm5, ymm4, ymm8);//d = c - (a*b) //B11[1][0-3] -= A11[1][0] * B11[0][0-3] - ymm11 = _mm256_fnmadd_pd(ymm6, ymm4, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][0] * B11[0][0-3] - ymm13 = _mm256_fnmadd_pd(ymm7, ymm4, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][0] * B11[0][0-3] - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - ymm11 = _mm256_fnmadd_pd(ymm9, ymm8, ymm11);//d = c - (a*b) //B11[2][0-3] -= A11[2][1] * B11[1][0-3] - ymm13 = _mm256_fnmadd_pd(ymm10, ymm8, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][1] * B11[1][0-3] - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - ymm13 = _mm256_fnmadd_pd(ymm12, ymm11, ymm13);//d = c - (a*b) //B11[3][0-3] -= A11[3][2] * B11[2][0-3] - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - ymm1 = _mm256_unpacklo_pd(ymm4, ymm8); //B11[0][0] B11[1][0] B11[0][2] B11[1][2] - ymm3 = _mm256_unpacklo_pd(ymm11, ymm13); //B11[2][0] B11[3][0] B11[2][2] B11[3][2] - - //rearrange low elements - ymm0 = _mm256_permute2f128_pd(ymm1,ymm3,0x20); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_permute2f128_pd(ymm1,ymm3,0x31); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - - ////unpackhigh//// - ymm14 = _mm256_unpackhi_pd(ymm4, ymm8); //B11[0][1] B11[1][1] B11[0][3] B11[1][3] - - ymm15 = _mm256_unpackhi_pd(ymm11, ymm13); //B11[2][1] B11[3][1] B11[2][3] B11[3][3] - - //rearrange high elements - ymm1 = _mm256_permute2f128_pd(ymm14,ymm15,0x20); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_permute2f128_pd(ymm14,ymm15,0x31); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) - - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) - - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - } - - } - if(m_remainder) //implementation for remainder rows(when 'M' is not a multiple of D_MR) - { - a10 = L +i; //pointer to block of A to be used for GEMM - a11 = L + i + (i*cs_a); //pointer to block of A to be used for TRSM - b01 = B + j*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j* cs_b; //pointer to block of B to be used for TRSM - - - k_iter = i / D_MR; //number of times GEMM operations to be performed - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to hold alpha value - - ///GEMM for previously calculated values /// - - - //load 4x4 block from b11 - if(n_remainder == 3) - { - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_broadcast_sd((double const *)&ones); - } - if(n_remainder == 2) - { - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_broadcast_sd((double const *)&ones); - ymm3 = _mm256_broadcast_sd((double const *)&ones); - } - if(n_remainder == 1) - { - ymm0 = _mm256_loadu_pd((double const *)(b11)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_broadcast_sd((double const *)&ones); - ymm2 = _mm256_broadcast_sd((double const *)&ones); - ymm3 = _mm256_broadcast_sd((double const *)&ones); - } - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_b01_dup = b01; - ymm8 = _mm256_loadu_pd((double const *)(a10)); //A10[0][0] A10[1][0] A10[2][0] A10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(a10 + cs_a)); //A10[0][1] A10[1][1] A10[2][1] A10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(a10 + cs_a * 2)); //A10[0][2] A10[1][2] A10[2][2] A10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(a10 + cs_a * 3)); //A10[0][3] A10[1][3] A10[2][3] A10[3][3] - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[0][3] - - b01 += 1; //move to next row of B - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B01[0][0]*A10[0][0] B01[0][0]*A10[1][0] B01[0][0]*A10[2][0] B01[0][0]*A10[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B01[0][1]*A10[0][0] B01[0][1]*A10[1][0] B01[0][1]*A10[2][0] B01[0][1]*A10[3][0]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B01[0][2]*A10[0][0] B01[0][2]*A10[1][0] B01[0][2]*A10[2][0] B01[0][2]*A10[3][0]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B01[0][3]*A10[0][0] B01[0][3]*A10[1][0] B01[0][3]*A10[2][0] B01[0][3]*A10[3][0]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[1][3] - - b01 += 1; //move to next row of B - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B01[1][0]*A10[0][1] B01[1][0]*A10[1][1] B01[1][0]*A10[2][1] B01[1][0]*A10[3][1]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B01[1][1]*A10[0][1] B01[1][1]*A10[1][1] B01[1][1]*A10[2][1] B01[1][1]*A10[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B01[1][2]*A10[0][1] B01[1][2]*A10[1][1] B01[1][2]*A10[2][1] B01[1][2]*A10[3][1]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B01[1][3]*A10[0][1] B01[1][3]*A10[1][1] B01[1][3]*A10[2][1] B01[1][3]*A10[3][1]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[2][3] - - b01 += 1; //move to next row of B - - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B01[2][0]*A10[0][2] B01[2][0]*A10[1][2] B01[2][0]*A10[2][2] B01[2][0]*A10[3][2]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B01[2][1]*A10[0][2] B01[2][1]*A10[1][2] B01[2][1]*A10[2][2] B01[2][1]*A10[3][2]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B01[2][2]*A10[0][2] B01[2][2]*A10[1][2] B01[2][2]*A10[2][2] B01[2][2]*A10[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B01[2][3]*A10[0][2] B01[2][3]*A10[1][2] B01[2][3]*A10[2][2] B01[2][3]*A10[3][2]) - - ymm12 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 0)); //B10[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 1)); //B10[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 2)); //B10[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(b01 + cs_b * 3)); //B10[3][3] - - b01 += 1; //move to next row of B - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B01[3][0]*A10[0][3] B01[3][0]*A10[1][3] B01[3][0]*A10[2][3] B01[3][0]*A10[3][3]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B01[3][1]*A10[0][3] B01[3][1]*A10[1][3] B01[3][1]*A10[2][3] B01[3][1]*A10[3][3]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B01[3][2]*A10[0][3] B01[3][2]*A10[1][3] B01[3][2]*A10[2][3] B01[3][2]*A10[3][3]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B01[3][3]*A10[0][3] B01[3][3]*A10[1][3] B01[3][3]*A10[2][3] B01[3][3]*A10[3][3]) - - a10 += D_MR * cs_a; //pointer math to find next block of A for GEMM - b01 = ptr_b01_dup + D_MR; //pointer math to find next block of B for GEMM - - } - - ymm8 = _mm256_fmsub_pd(ymm0, ymm16, ymm4); //B11[0-3][0] * alpha -= ymm4 - ymm9 = _mm256_fmsub_pd(ymm1, ymm16, ymm5); //B11[0-3][1] * alpha -= ymm5 - ymm10 = _mm256_fmsub_pd(ymm2, ymm16, ymm6); //B11[0-3][2] * alpha -= ymm6 - ymm11 = _mm256_fmsub_pd(ymm3, ymm16, ymm7); //B11[0-3][3] * alpha -= ymm7 - - ///implement TRSM/// - //determine correct values to store - if(m_remainder == 3) - { - ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x08); - ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x08); - ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x08); - ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x08); - - } - if(m_remainder == 2) - { - ymm0 = _mm256_permute2f128_pd(ymm8, ymm0, 0x30); - ymm1 = _mm256_permute2f128_pd(ymm9, ymm1, 0x30); - ymm2 = _mm256_permute2f128_pd(ymm10, ymm2, 0x30); - ymm3 = _mm256_permute2f128_pd(ymm11, ymm3, 0x30); - - } - if(m_remainder == 1) - { - ymm0 = _mm256_blend_pd(ymm8, ymm0, 0x0E); - ymm1 = _mm256_blend_pd(ymm9, ymm1, 0x0E); - ymm2 = _mm256_blend_pd(ymm10, ymm2, 0x0E); - ymm3 = _mm256_blend_pd(ymm11, ymm3, 0x0E); - } - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b * 2), ymm2); //store(B11[0-3][2]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + (cs_b)), ymm1); //store(B11[0-3][1]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[0-3][0]) - } - - ///scalar code for trsm without alpha/// - dtrsm_small_AlXB_unitDiag(a11, b11, m_remainder, n_remainder, cs_a, cs_b); - } - } - return BLIS_SUCCESS; -} - - -/*implements TRSM for the case XA = alpha * B - *A is upper triangular, non-unit diagonal, no transpose - *dimensions: X:mxn A:nxn B: mxn - */ - -/* b11---> a01 ----> - ***************** *********** - *b01*b11* * * * * * * -b11 * * * * * **a01 * * a11 - | ***************** ********* | - | * * * * * *a11* * | - | * * * * * * * * | - v ***************** ****** v - * * * * * * * - * * * * * * * - ***************** * * - * - -*/ -static err_t bli_dtrsm_small_XAuB( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - dim_t D_MR = 8; //block dimension along the rows - dim_t D_NR = 4; //block dimension along the columns - - dim_t m = bli_obj_length(b); //number of rows - dim_t n = bli_obj_width(b); //number of columns - dim_t m_remainder = m % D_MR; //number of corner rows - dim_t n_remainder = n % D_NR; //number of corner columns - dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A - dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - - if(max(m,n)>150 && (m/n) < 22) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - - dim_t i, j, k; //loop variablse - dim_t k_iter; //determines the number of GEMM operations to be done - dim_t cs_b_offset[2]; //pre-calculated strides - - double ones = 1.0; - - double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha - double *L = a->buffer; //pointer to matrix A - double *B = b->buffer; //pointer to matrix B - - double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks - double *ptr_a01_dup; - - cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; - cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; - - //ymm scratch reginsters - __m256d ymm0, ymm1, ymm2, ymm3; - __m256d ymm4, ymm5, ymm6, ymm7; - __m256d ymm8, ymm9, ymm10, ymm11; - __m256d ymm12, ymm13, ymm14, ymm15; - __m256d ymm16; - - for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction - { - for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction - { - a01 = L + j*cs_a; //pointer to block of A to be used in GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i; //pointer to block of B to be used in GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] - - a01 += 1; //move to next row - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] - - a01 += 1; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] - - a01 += 1; //move to next row of A01 - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] - - a01 += 1; //move to next row of A01 - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - //load 8x4 block of B11 - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] - - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 - - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - ymm7 = _mm256_broadcast_sd((double const *)(&ones)); - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - //2nd col - a11 += cs_a; - ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] - - //3rd col - a11 += cs_a; - ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] - ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] - - //4th col - a11 += cs_a; - ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] - - //compute reciprocals of L(i,i) and broadcast in registers - ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] - ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] - - ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) - - ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] - - //extract a00 - ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) - - ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] - - ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] - - //extract a11 - ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) - - //(Row1): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] - ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] - ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] - - ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] - ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] - ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] - - ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] - - ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] - - //extract a22 - ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) - - //(Row2)FMA operations - ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] - ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] - - ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] - ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] - - ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] - - ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] - - //extract a33 - ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - - //(Row3)FMA operations - ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] - - ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] - - ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] - - ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] - - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) - } - if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) - { - a01 = L + j*cs_a; //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) - - ///load 4x4 block of b11 - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation begins/// - - for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] - - a01 += 1; //move to next row of A - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] - - a01 += 1; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] - - a01 += 1; //move to next row of A - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] - - a01 += 1; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - - //subtract the calculated GEMM block from current TRSM block - //load 8x4 block of B11 - if(n_remainder == 3) - { - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] - ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - } - if(n_remainder == 2) - { - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] - ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - } - if(n_remainder == 1) - { - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] - ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] - ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] - ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - } - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - ymm7 = _mm256_broadcast_sd((double const *)(&ones)); - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - //2nd col - a11 += cs_a; - ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] - - //3rd col - a11 += cs_a; - ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] - ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] - - //4th col - a11 += cs_a; - ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] - - //compute reciprocals of L(i,i) and broadcast in registers - ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] - ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] - - ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) - - ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] - - //extract a00 - ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) - - ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] - - ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] - - //extract a11 - ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) - - //(Row1): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] - ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] - ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] - - ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] - ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] - ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] - - ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] - - ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] - - //extract a22 - ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) - - //(Row2)FMA operations - ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] - ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] - - ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] - ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] - - ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] - - ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] - - //extract a33 - ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - - //(Row3)FMA operations - ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] - - ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] - - ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] - - ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - } - } - } - if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) - { - for(j = 0; (j+D_NR-1) a01 ----> - ***************** *********** - *b01*b11* * * * * * * -b11 * * * * * **a01 * * a11 - | ***************** ********* | - | * * * * * *a11* * | - | * * * * * * * * | - v ***************** ****** v - * * * * * * * - * * * * * * * - ***************** * * - * - -*/ - -static err_t bli_dtrsm_small_XAuB_unitDiag( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - dim_t D_MR = 8; //block dimension along the rows - dim_t D_NR = 4; //block dimension along the columns - - dim_t m = bli_obj_length(b); //number of rows - dim_t n = bli_obj_width(b); //number of columns - dim_t m_remainder = m % D_MR; //number of corner rows - dim_t n_remainder = n % D_NR; //number of corner columns - dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A - dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - - if((max(m,n)>180) && (m/n)<22) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - - dim_t i, j, k; //loop variablse - dim_t k_iter; //determines the number of GEMM operations to be done - dim_t cs_b_offset[2]; //pre-calculated strides - - double ones = 1.0; - - double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha - double *L = a->buffer; //pointer to matrix A - double *B = b->buffer; //pointer to matrix B - - double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks - double *ptr_a01_dup; - - cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; - cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; - - //ymm scratch reginsters - __m256d ymm0, ymm1, ymm2, ymm3; - __m256d ymm4, ymm5, ymm6, ymm7; - __m256d ymm8, ymm9, ymm10, ymm11; - __m256d ymm12, ymm13, ymm14, ymm15; - __m256d ymm16; - - for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction - { - for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction - { - a01 = L + j*cs_a; //pointer to block of A to be used in GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i; //pointer to block of B to be used in GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] - - a01 += 1; //move to next row - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] - - a01 += 1; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] - - a01 += 1; //move to next row of A01 - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] - - a01 += 1; //move to next row of A01 - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - //load 8x4 block of B11 - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] - - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 - - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - ymm7 = _mm256_broadcast_sd((double const *)(&ones)); - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - //2nd col - a11 += cs_a; - ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] - - //3rd col - a11 += cs_a; - ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] - ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] - - //4th col - a11 += cs_a; - ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] - - ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] - - //(Row1): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] - ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] - ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] - - ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] - ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] - ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] - - - //(Row2)FMA operations - ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] - ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] - - ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] - ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] - - - //(Row3)FMA operations - ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] - - ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] - - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) - } - if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) - { - a01 = L + j*cs_a; //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) - - ///load 4x4 block of b11 - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation begins/// - - for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] - - a01 += 1; //move to next row of A - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] - - a01 += 1; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] - - a01 += 1; //move to next row of A - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] - - a01 += 1; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - - //subtract the calculated GEMM block from current TRSM block - //load 8x4 block of B11 - if(n_remainder == 3) - { - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] - ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - } - if(n_remainder == 2) - { - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] - ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - } - if(n_remainder == 1) - { - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] - ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] - ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] - ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - } - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - ymm7 = _mm256_broadcast_sd((double const *)(&ones)); - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - //2nd col - a11 += cs_a; - ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][1] - - //3rd col - a11 += cs_a; - ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][2] - ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][2] - - //4th col - a11 += cs_a; - ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[3][3] - - ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][3] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[1][3] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[2][3] - - - //(Row1): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] - ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] - ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] - - ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] - ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] - ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] - - //(Row2)FMA operations - ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] - ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] - - ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] - ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] - - //(Row3)FMA operations - ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] - - ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] - - ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] - - ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - } - } - } - if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) - { - for(j = 0; (j+D_NR-1) a01 ----> - ***************** *********** - *b01*b11* * * * * * * -b11 * * * * * **a01 * * a11 - | ***************** ********* | - | * * * * * *a11* * | - | * * * * * * * * | - v ***************** ****** v - * * * * * * * - * * * * * * * - ***************** * * - * - -*/ -static err_t bli_dtrsm_small_XAltB( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - dim_t D_MR = 8; //block dimension along the rows - dim_t D_NR = 4; //block dimension along the columns - - dim_t m = bli_obj_length(b); //number of rows - dim_t n = bli_obj_width(b); //number of columns - dim_t m_remainder = m % D_MR; //number of corner rows - dim_t n_remainder = n % D_NR; //number of corner columns - dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A - dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - - if(max(m,n) > 120) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - - dim_t i, j, k; //loop variablse - dim_t k_iter; //determines the number of GEMM operations to be done - dim_t cs_b_offset[2]; //pre-calculated strides - - double ones = 1.0; - - double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha - double *L = a->buffer; //pointer to matrix A - double *B = b->buffer; //pointer to matrix B - - double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks - double *ptr_a01_dup; - - cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; - cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; - - //ymm scratch reginsters - __m256d ymm0, ymm1, ymm2, ymm3; - __m256d ymm4, ymm5, ymm6, ymm7; - __m256d ymm8, ymm9, ymm10, ymm11; - __m256d ymm12, ymm13, ymm14, ymm15; - __m256d ymm16; - - for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction - { - for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction - { - a01 = L + j; //pointer to block of A to be used in GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i; //pointer to block of B to be used in GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] - - a01 += cs_a; //move to next row - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] - - a01 += cs_a; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] - - a01 += cs_a; //move to next row of A01 - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] - - a01 += cs_a; //move to next row of A01 - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - //load 8x4 block of B11 - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] - - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 - - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - ymm7 = _mm256_broadcast_sd((double const *)(&ones)); - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - //2nd col - a11 += 1; - ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] - - //3rd col - a11 += 1; - ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] - - //4th col - a11 += 1; - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] - - //compute reciprocals of L(i,i) and broadcast in registers - ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] - ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] - - ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) - - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] - - //extract a00 - ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) - - ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] - - ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] - - //extract a11 - ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) - - //(Row1): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] - ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] - ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] - - ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] - ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] - ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] - - ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] - - ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] - - //extract a22 - ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) - - //(Row2)FMA operations - ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] - ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] - - ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] - ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] - - ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] - - ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] - - //extract a33 - ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - - //(Row3)FMA operations - ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] - - ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] - - ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] - - ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] - - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) - } - if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) - { - a01 = L + j; //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) - - ///load 4x4 block of b11 - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation begins/// - - for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] - - a01 += cs_a; //move to next row of A - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] - - a01 += cs_a; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] - - a01 += cs_a; //move to next row of A - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] - - a01 += cs_a; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - - //subtract the calculated GEMM block from current TRSM block - //load 8x4 block of B11 - if(n_remainder == 3) - { - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] - ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - } - if(n_remainder == 2) - { - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] - ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - } - if(n_remainder == 1) - { - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] - ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] - ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] - ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - } - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - ymm7 = _mm256_broadcast_sd((double const *)(&ones)); - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - //2nd col - a11 += 1; - ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] - - //3rd col - a11 += 1; - ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] - - //4th col - a11 += 1; - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] - - //compute reciprocals of L(i,i) and broadcast in registers - ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] - ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] - - ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) - - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] - - //extract a00 - ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) - - ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] - - ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] - - //extract a11 - ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) - - //(Row1): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] - ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] - ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] - - ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] - ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] - ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] - - ymm9 = _mm256_mul_pd(ymm9, ymm0); //B11[0-3][1] /= A11[1][1] - - ymm13 = _mm256_mul_pd(ymm13, ymm0); //B11[4-7][1] /= A11[1][1] - - //extract a22 - ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) - - //(Row2)FMA operations - ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] - ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] - - ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] - ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] - - ymm10 = _mm256_mul_pd(ymm10, ymm0); //B11[0-3][2] /= A11[2][2] - - ymm14 = _mm256_mul_pd(ymm14, ymm0); //B11[4-7][2] /= A11[2][2] - - //extract a33 - ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - - //(Row3)FMA operations - ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] - - ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] - - ymm11 = _mm256_mul_pd(ymm11, ymm0); //B11[0-3][3] /= A11[3][3] - - ymm15 = _mm256_mul_pd(ymm15, ymm0); //B11[4-7][3] /= A11[3][3] - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - } - } - } - if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) - { - for(j = 0; (j+D_NR-1) a01 ----> - ***************** *********** - *b01*b11* * * * * * * -b11 * * * * * **a01 * * a11 - | ***************** ********* | - | * * * * * *a11* * | - | * * * * * * * * | - v ***************** ****** v - * * * * * * * - * * * * * * * - ***************** * * - * - -*/ -static err_t bli_dtrsm_small_XAltB_unitDiag( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - dim_t D_MR = 8; //block dimension along the rows - dim_t D_NR = 4; //block dimension along the columns - - dim_t m = bli_obj_length(b); //number of rows - dim_t n = bli_obj_width(b); //number of columns - dim_t m_remainder = m % D_MR; //number of corner rows - dim_t n_remainder = n % D_NR; //number of corner columns - dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A - dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - - if(max(m,n) > 120) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - - - dim_t i, j, k; //loop variablse - dim_t k_iter; //determines the number of GEMM operations to be done - dim_t cs_b_offset[2]; //pre-calculated strides - - double ones = 1.0; - - double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha - double *L = a->buffer; //pointer to matrix A - double *B = b->buffer; //pointer to matrix B - - double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks - double *ptr_a01_dup; - - cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; - cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; - - //ymm scratch reginsters - __m256d ymm0, ymm1, ymm2, ymm3; - __m256d ymm4, ymm5, ymm6, ymm7; - __m256d ymm8, ymm9, ymm10, ymm11; - __m256d ymm12, ymm13, ymm14, ymm15; - __m256d ymm16; - - for(i = 0; (i+D_MR-1) < m; i += D_MR) //loop along 'M' direction - { - for(j = 0; (j+D_NR-1) < n; j += D_NR) //loop along 'N' direction - { - a01 = L + j; //pointer to block of A to be used in GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i; //pointer to block of B to be used in GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = j / D_NR; //number of GEMM operations to be done(in blocks of 4x4) - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] - - a01 += cs_a; //move to next row - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] - - a01 += cs_a; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] - - a01 += cs_a; //move to next row of A01 - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] - - a01 += cs_a; //move to next row of A01 - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - //load 8x4 block of B11 - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] - - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 - - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - //2nd col - a11 += 1; - ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] - - //3rd col - a11 += 1; - ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] - - //4th col - a11 += 1; - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] - - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] - - //(Row1): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] - ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] - ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] - - ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] - ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] - ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] - - //(Row2)FMA operations - ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] - ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] - - ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] - ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] - - //(Row3)FMA operations - ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] - - ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] - - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) - } - if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) - { - a01 = L + j; //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = j / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) - - ///load 4x4 block of b11 - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation begins/// - - for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] - - a01 += cs_a; //move to next row of A - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] - - a01 += cs_a; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] - - a01 += cs_a; //move to next row of A - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] - - a01 += cs_a; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - - //subtract the calculated GEMM block from current TRSM block - //load 8x4 block of B11 - if(n_remainder == 3) - { - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][2] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][2] - ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - } - if(n_remainder == 2) - { - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0-3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][1] - ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - } - if(n_remainder == 1) - { - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0-3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4-7][0] - ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] - ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] - ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm11 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm15 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - } - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - ymm7 = _mm256_broadcast_sd((double const *)(&ones)); - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - //2nd col - a11 += 1; - ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] - - //3rd col - a11 += 1; - ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] - - //4th col - a11 += 1; - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] - - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] - - //(Row1): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm1, ymm8, ymm9); //B11[0-3][1] -= B11[0-3][0] * A11[0][1] - ymm10 = _mm256_fnmadd_pd(ymm3, ymm8, ymm10); //B11[0-3][2] -= B11[0-3][0] * A11[0][2] - ymm11 = _mm256_fnmadd_pd(ymm2, ymm8, ymm11); //B11[0-3][3] -= B11[0-3][0] * A11[0][3] - - ymm13 = _mm256_fnmadd_pd(ymm1, ymm12, ymm13); //B11[4-7][1] -= B11[4-7][0] * A11[0][1] - ymm14 = _mm256_fnmadd_pd(ymm3, ymm12, ymm14); //B11[4-7][2] -= B11[4-7][0] * A11[0][2] - ymm15 = _mm256_fnmadd_pd(ymm2, ymm12, ymm15); //B11[4-7][3] -= B11[4-7][0] * A11[0][3] - - //(Row2)FMA operations - ymm10 = _mm256_fnmadd_pd(ymm4, ymm9, ymm10); //B11[0-3][2] -= B11[0-3][1] * A11[1][2] - ymm11 = _mm256_fnmadd_pd(ymm5, ymm9, ymm11); //B11[0-3][3] -= B11[0-3][1] * A11[1][3] - - ymm14 = _mm256_fnmadd_pd(ymm4, ymm13, ymm14); //B11[4-7][2] -= B11[4-7][1] * A11[1][2] - ymm15 = _mm256_fnmadd_pd(ymm5, ymm13, ymm15); //B11[4-7][3] -= B11[4-7][1] * A11[1][3] - - //(Row3)FMA operations - ymm11 = _mm256_fnmadd_pd(ymm6, ymm10, ymm11); //B11[0-3][3] -= B11[0-3][2] * A11[2][3] - - ymm15 = _mm256_fnmadd_pd(ymm6, ymm14, ymm15); //B11[4-7][3] -= B11[4-7][2] * A11[2][3] - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - } - } - } - if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) - { - for(j = 0; (j+D_NR-1) 120) - return BLIS_NOT_YET_IMPLEMENTED; - - dim_t i, j, k; //loop variablse - dim_t k_iter; //determines the number of GEMM operations to be done - dim_t cs_b_offset[2]; //pre-calculated strides - - double ones = 1.0; - - double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha - double *L = a->buffer; //pointer to matrix A - double *B = b->buffer; //pointer to matrix B - - double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks - double *ptr_a01_dup; - - cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; - cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; - - //ymm scratch reginsters - __m256d ymm0, ymm1, ymm2, ymm3; - __m256d ymm4, ymm5, ymm6, ymm7; - __m256d ymm8, ymm9, ymm10, ymm11; - __m256d ymm12, ymm13, ymm14, ymm15; - __m256d ymm16; - - for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction - { - for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction - { - a01 = L + j*cs_a +(j+D_NR); //pointer to block of A to be used in GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM - b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] - - a01 += 1; //move to next row - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] - - a01 += 1; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] - - a01 += 1; //move to next row of A01 - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] - - a01 += 1; //move to next row of A01 - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - //load 8x4 block of B11 - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] - - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 - - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - ymm7 = _mm256_broadcast_sd((double const *)(&ones)); - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - //2nd col - a11 += 1; - ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] - - //3rd col - a11 += 1; - ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] - - //4th col - a11 += 1; - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] - - //compute reciprocals of L(i,i) and broadcast in registers - ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] - ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] - - ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) - - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] - - //extract a33 - ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - - ymm11 = _mm256_mul_pd(ymm11, ymm0); - - ymm15 = _mm256_mul_pd(ymm15, ymm0); - - //extract a22 - ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) - - //(row 3):FMA operations - ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); - ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); - - ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); - ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); - - ymm10 = _mm256_mul_pd(ymm10, ymm0); - - ymm14 = _mm256_mul_pd(ymm14, ymm0); - - //extract a11 - ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) - - //(Row 2): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); - - ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); - - ymm9 = _mm256_mul_pd(ymm9, ymm0); - - ymm13 = _mm256_mul_pd(ymm13, ymm0); - - //extract a00 - ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) - - //(Row 1): FMA operations - ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); - - ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); - - ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] - - ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] - - - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) - - - } - if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) - { - a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j + D_NR)*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) - - ///load 4x4 block of b11 - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation begins/// - - for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] - - a01 += 1; //move to next row of A - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] - - a01 += 1; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] - - a01 += 1; //move to next row of A - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] - - a01 += 1; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - - //subtract the calculated GEMM block from current TRSM block - //load 8x4 block of B11 - if(n_remainder == 3) - { - ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] - } - if(n_remainder == 2) - { - ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] - } - if(n_remainder == 1) - { - ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] - ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] - ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] - } - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - ymm7 = _mm256_broadcast_sd((double const *)(&ones)); - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - //2nd col - a11 += 1; - ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] - - //3rd col - a11 += 1; - ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] - - //4th col - a11 += 1; - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] - - //compute reciprocals of L(i,i) and broadcast in registers - ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] - ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[1][1] A11[3][3] - - ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm7 = _mm256_div_pd(ymm7, ymm0); //(1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3]) - - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] - - //extract a33 - ymm0 = _mm256_permute_pd(ymm7, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - - ymm11 = _mm256_mul_pd(ymm11, ymm0); - - ymm15 = _mm256_mul_pd(ymm15, ymm0); - - //extract a22 - ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x11);//(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) - - //(row 3):FMA operations - ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); - ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); - - ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); - ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); - - ymm10 = _mm256_mul_pd(ymm10, ymm0); - - ymm14 = _mm256_mul_pd(ymm14, ymm0); - - //extract a11 - ymm0 = _mm256_permute_pd(ymm7, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00);//(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) - - //(Row 2): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); - - ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); - - ymm9 = _mm256_mul_pd(ymm9, ymm0); - - ymm13 = _mm256_mul_pd(ymm13, ymm0); - - //extract a00 - ymm0 = _mm256_permute_pd(ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm0 = _mm256_permute2f128_pd(ymm0, ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) - - //(Row 1): FMA operations - ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); - - ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); - - ymm8 = _mm256_mul_pd(ymm8, ymm0); //B11[0-3][0] /= A11[0][0] - - ymm12 = _mm256_mul_pd(ymm12, ymm0); //B11[4-7][0] /= A11[0][0] - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) - } - } - } - if(i<0) - i += D_NR; - if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) - { - for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction - { - a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) - - ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha - ///GEMM for previous blocks /// - - ///load 4x4 block of b11 - ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - //multiply by alpha - ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha - ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha - ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha - ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //load 4x4 bblock of b10 - ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] - - //broadcast 1st row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) - - //braodcast 3rd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) - - //broadcast 4th row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) - - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM - } - - ///GEMM code end/// - - ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 - ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 - ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 - ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] - ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] - - //2nd col - a11 += cs_a; - ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] - - //3rd col - a11 += cs_a; - ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] - ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] - - //4th col - a11 += cs_a; - ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] - - ymm14 = _mm256_broadcast_sd((double const *)&ones); - - //compute reciprocals of A(i,i) and broadcast in registers - ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] - ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] - - ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] - - //extract a33 - ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - - ymm3 = _mm256_mul_pd(ymm3, ymm15); - - //extract a22 - ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) - - //(Row 3): FMA operations - ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); - ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); - - ymm2 = _mm256_mul_pd(ymm2, ymm15); - - //extract a11 - ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) - - //(ROW 2): FMA operations - ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); - - ymm1 = _mm256_mul_pd(ymm1, ymm15); - - //extract A00 - ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) - - //(Row 1):FMA operations - ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); - - ymm0 = _mm256_mul_pd(ymm0, ymm15); - - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) - - } - if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) - { - a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointwr to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value - ///GEMM for previous blocks /// - - ///load 4x4 block of b11 - if(n_remainder == 3) - { - ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - } - if(n_remainder == 2) - { - ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - } - if(n_remainder == 1) - { - ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - } - //multiply by alpha - ymm0 = _mm256_mul_pd(ymm0, ymm16); //B11[x][0] *= alpha - ymm1 = _mm256_mul_pd(ymm1, ymm16); //B11[x][1] *=alpha - ymm2 = _mm256_mul_pd(ymm2, ymm16); //B11[x][2] *= alpha - ymm3 = _mm256_mul_pd(ymm3, ymm16); //B11[x][3] *= alpha - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - - ///GEMM processing stars/// - - for(k = 0; k < k_iter; k++) - { - ptr_a01_dup = a01; - - //load 4x4 bblock of b10 - ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] - - //broadcast 1st row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) - - //braodcast 3rd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) - - //broadcast 4th row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) - - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM - - } - - ///GEMM code ends/// - - ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -= ymm4 - ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 - ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 - ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] - ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] - - //2nd col - a11 += cs_a; - ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] - - //3rd col - a11 += cs_a; - ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] - ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] - - //4th col - a11 += cs_a; - ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] - - ymm14 = _mm256_broadcast_sd((double const *)&ones); - - //compute reciprocals of A(i,i) and broadcast in registers - ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] - ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] - - ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] - - //extract a33 - ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - - ymm3 = _mm256_mul_pd(ymm3, ymm15); - - //extract a22 - ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) - - //(Row 3): FMA operations - ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); - ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); - - ymm2 = _mm256_mul_pd(ymm2, ymm15); - - //extract a11 - ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) - - //(ROW 2): FMA operations - ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); - - ymm1 = _mm256_mul_pd(ymm1, ymm15); - - //extract A00 - ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) - - //(Row 1):FMA operations - ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); - - ymm0 = _mm256_mul_pd(ymm0, ymm15); - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) - _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) - } - - } - m_remainder -= 4; - i -= 4; - } -// if(i < 0) i = 0; - if(m_remainder) ///implementation for remainder rows - { - dtrsm_small_XAlB(L, B, AlphaVal, m_remainder, n, cs_a, cs_b); - } - return BLIS_SUCCESS; -} - -/*implements TRSM for the case XA = alpha * B - *A is lower triangular, unit-diagonal, no transpose - *dimensions: X:mxn A:nxn B: mxn - */ - -/* <---b11 <---a11 - ***************** * - *b01*b11* * * * * - ^ * * * * * ^ * * - | ***************** | ******* - | * * * * * | * * * - | * * * * * a01* * * -b10 ***************** ************* - * * * * * * * * * - * * * * * * * * * - ***************** ******************* - -*/ -static err_t bli_dtrsm_small_XAlB_unitDiag( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - dim_t D_MR = 8; //block dimension along the rows - dim_t D_NR = 4; //block dimension along the columns - - dim_t m = bli_obj_length(b); //number of rows - dim_t n = bli_obj_width(b); //number of columns - dim_t m_remainder = m % D_MR; //number of corner rows - dim_t n_remainder = n % D_NR; //number of corner columns - dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A - dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - - if(max(m,n) > 120) - return BLIS_NOT_YET_IMPLEMENTED; - - dim_t i, j, k; //loop variablse - dim_t k_iter; //determines the number of GEMM operations to be done - dim_t cs_b_offset[2]; //pre-calculated strides - - double ones = 1.0; - - double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha - double *L = a->buffer; //pointer to matrix A - double *B = b->buffer; //pointer to matrix B - - double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks - double *ptr_a01_dup; - - cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; - cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; - - //ymm scratch reginsters - __m256d ymm0, ymm1, ymm2, ymm3; - __m256d ymm4, ymm5, ymm6, ymm7; - __m256d ymm8, ymm9, ymm10, ymm11; - __m256d ymm12, ymm13, ymm14, ymm15; - __m256d ymm16; - - for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction - { - for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction - { - a01 = L + j*cs_a +(j+D_NR); //pointer to block of A to be used in GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM - b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] - - a01 += 1; //move to next row - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] - - a01 += 1; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] - - a01 += 1; //move to next row of A01 - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] - - a01 += 1; //move to next row of A01 - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - //load 8x4 block of B11 - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] - - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 - - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - //2nd col - a11 += 1; - ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] - - //3rd col - a11 += 1; - ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] - - //4th col - a11 += 1; - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] - - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] - - //(row 3):FMA operations - ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); - ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); - - ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); - ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); - - //(Row 2): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); - - ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); - - //(Row 1): FMA operations - ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); - - ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); - - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[4-7][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b), ymm11); //store(B11[0-3][3]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + cs_b + D_NR), ymm15);//store(B11[4-7][3]) - - - } - if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) - { - a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j + D_NR)*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be performed(in blocks of 4x4) - - ///load 4x4 block of b11 - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation begins/// - - for(k = 0; k < k_iter; k++) ///loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] - - a01 += 1; //move to next row of A - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR));//B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] - - a01 += 1; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] - - a01 += 1; //move to next row of A - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] - - a01 += 1; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - - //subtract the calculated GEMM block from current TRSM block - //load 8x4 block of B11 - if(n_remainder == 3) - { - ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] - } - if(n_remainder == 2) - { - ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] - } - if(n_remainder == 1) - { - ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] - ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] - ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] - } - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= B10[0-3][0] - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha -= B10[4-7][0] - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha -= B10[0-3][1] - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= B10[4-7][1] - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= B10[0-3][2] - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= B10[4-7][2] - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= B10[0-3][3] - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= B10[4-7][3] - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - ymm7 = _mm256_broadcast_sd((double const *)(&ones)); - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - //2nd col - a11 += 1; - ymm1 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][1] - - //3rd col - a11 += 1; - ymm3 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][2] - ymm4 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][2] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][2] - - //4th col - a11 += 1; - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 3)); //A11[3][3] - - - ymm2 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 0)); //A11[0][3] - ymm5 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 1)); //A11[1][3] - ymm6 = _mm256_broadcast_sd((double const *)(a11+ cs_a * 2)); //A11[2][3] - - //(row 3):FMA operations - ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); - ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); - - ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); - ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); - - //(Row 2): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); - - ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); - - //(Row 1): FMA operations - ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); - - ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) - } - } - } - if(i<0) - i += D_NR; - if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) - { - for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction - { - a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) - - ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha - ///GEMM for previous blocks /// - - ///load 4x4 block of b11 - ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - //multiply by alpha - ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha - ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha - ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha - ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //load 4x4 bblock of b10 - ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] - - //broadcast 1st row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) - - //braodcast 3rd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) - - //broadcast 4th row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) - - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM - } - - ///GEMM code end/// - - ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 - ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 - ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 - ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] - ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] - - //2nd col - a11 += cs_a; - ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] - - //3rd col - a11 += cs_a; - ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] - ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] - - //4th col - a11 += cs_a; - ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] - - ymm14 = _mm256_broadcast_sd((double const *)&ones); - - - //(Row 3): FMA operations - ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); - ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); - - //(ROW 2): FMA operations - ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); - - //(Row 1):FMA operations - ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); - - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //store(B11[x][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) - - } - if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) - { - a01 = L + j*cs_a + (j+D_NR); //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointwr to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha value - ///GEMM for previous blocks /// - - ///load 4x4 block of b11 - if(n_remainder == 3) - { - ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - } - if(n_remainder == 2) - { - ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - } - if(n_remainder == 1) - { - ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - } - //multiply by alpha - ymm0 = _mm256_mul_pd(ymm0, ymm16); //B11[x][0] *= alpha - ymm1 = _mm256_mul_pd(ymm1, ymm16); //B11[x][1] *=alpha - ymm2 = _mm256_mul_pd(ymm2, ymm16); //B11[x][2] *= alpha - ymm3 = _mm256_mul_pd(ymm3, ymm16); //B11[x][3] *= alpha - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - - ///GEMM processing stars/// - - for(k = 0; k < k_iter; k++) - { - ptr_a01_dup = a01; - - //load 4x4 bblock of b10 - ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] - - //broadcast 1st row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[0][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[1][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) - - //braodcast 3rd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[2][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) - - //broadcast 4th row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 0)); //A01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 1)); //A01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 2)); //A01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + cs_a * 3)); //A01[3][3] - - a01 += 1; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) - - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR; //pointer math to find next block of A for GEMM - - } - - ///GEMM code ends/// - - ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -= ymm4 - ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 - ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 - ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][0] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][0] - ymm7 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][0] - - //2nd col - a11 += cs_a; - ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][1] - - //3rd col - a11 += cs_a; - ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][2] - ymm12 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[1][2] - - //4th col - a11 += cs_a; - ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][3] - - ymm14 = _mm256_broadcast_sd((double const *)&ones); - - //(Row 3): FMA operations - ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); - ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); - - //(ROW 2): FMA operations - ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); - - //(Row 1):FMA operations - ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) - _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) - } - - } - m_remainder -= 4; - i -= 4; - } - if(m_remainder) - { - dtrsm_small_XAlB_unitDiag(L, B, AlphaVal, m_remainder, n, cs_a, cs_b); - } - return BLIS_SUCCESS; -} - - -/*implements TRSM for the case XA = alpha * B - *A is lower triangular, non-unit diagonal, no transpose - *dimensions: X:mxn A:nxn B: mxn - */ - -/* <---b11 <---a11 - ***************** * - *b01*b11* * * * * - ^ * * * * * ^ * * - | ***************** | ******* - | * * * * * | * * * - | * * * * * a01* * * -b10 ***************** ************* - * * * * * * * * * - * * * * * * * * * - ***************** ******************* - -*/ -static err_t bli_dtrsm_small_XAutB( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - dim_t D_MR = 8; //block dimension along the rows - dim_t D_NR = 4; //block dimension along the columns - - dim_t m = bli_obj_length(b); //number of rows - dim_t n = bli_obj_width(b); //number of columns - dim_t m_remainder = m % D_MR; //number of corner rows - dim_t n_remainder = n % D_NR; //number of corner columns - dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A - dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - - if(max(m,n) > 120) - return BLIS_NOT_YET_IMPLEMENTED; - - dim_t i, j, k; //loop variablse - dim_t k_iter; //determines the number of GEMM operations to be done - dim_t cs_b_offset[2]; //pre-calculated strides - - double ones = 1.0; - - double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha - double *L = a->buffer; //pointer to matrix A - double *B = b->buffer; //pointer to matrix B - - double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks - double *ptr_a01_dup; - - cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; - cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; - - //ymm scratch reginsters - __m256d ymm0, ymm1, ymm2, ymm3; - __m256d ymm4, ymm5, ymm6, ymm7; - __m256d ymm8, ymm9, ymm10, ymm11; - __m256d ymm12, ymm13, ymm14, ymm15; - __m256d ymm16; - - for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction - { - for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction - { - a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM - b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] - - a01 += cs_a; //move to next row - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] - - a01 += cs_a; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] - - a01 += cs_a; //move to next row of A01 - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] - - a01 += cs_a; //move to next row of A01 - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - //load 8x4 block of B11 - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 - - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - a11 += cs_a; - - //2nd col - ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - - a11 += cs_a; - - //3rd col - ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - - a11 += cs_a; - - //4th col - ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] - - - ymm7 = _mm256_broadcast_sd((double const *)&ones); - - //compute reciprocals of A(i,i) and broadcast in registers - ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] - ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] - - ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm0 = _mm256_div_pd(ymm7, ymm0); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] - - ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - //extract a33 - ymm7 = _mm256_permute_pd(ymm0, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) - ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - - ymm11 = _mm256_mul_pd(ymm11, ymm7); - - ymm15 = _mm256_mul_pd(ymm15, ymm7); - - //extract a22 - ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) - - //(Row 3): FMA operations - ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); - ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); - - //(Row 3): FMA operations - ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); - ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); - - ymm10 = _mm256_mul_pd(ymm10, ymm7); - - ymm14 = _mm256_mul_pd(ymm14, ymm7); - - //extract a11 - ymm7 = _mm256_permute_pd(ymm0, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) - ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) - - //(ROW 2): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); - - ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); - - ymm9 = _mm256_mul_pd(ymm9, ymm7); - - ymm13 = _mm256_mul_pd(ymm13, ymm7); - - //extract A00 - ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) - - //(Row 1):FMA operations - ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); - - ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); - - ymm8 = _mm256_mul_pd(ymm8, ymm7); - - ymm12 = _mm256_mul_pd(ymm12, ymm7); - - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[x][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[x][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[x][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[x][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //(store(B11[x][2])) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //(store(B11[x][2])) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[x][3]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[x][3]) - - } - if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) - { - - a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM - b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] - - a01 += cs_a; //move to next row - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] - - a01 += cs_a; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] - - a01 += cs_a; //move to next row of A01 - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] - - a01 += cs_a; //move to next row of A01 - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - //load 8x4 block of B11 - if(n_remainder == 3) - { - ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] - } - if(n_remainder == 2) - { - ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] - } - if(n_remainder == 1) - { - ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] - ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] - ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] - } - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 - - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - a11 += cs_a; - - //2nd col - ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - - a11 += cs_a; - - //3rd col - ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - - a11 += cs_a; - - //4th col - ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] - - - ymm7 = _mm256_broadcast_sd((double const *)&ones); - - //compute reciprocals of A(i,i) and broadcast in registers - ymm0 = _mm256_unpacklo_pd(ymm0, ymm2); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] - ymm2 = _mm256_unpacklo_pd(ymm5, ymm6); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] - - ymm0 = _mm256_blend_pd(ymm0, ymm2, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm0 = _mm256_div_pd(ymm7, ymm0); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] - - ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - //extract a33 - ymm7 = _mm256_permute_pd(ymm0, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) - ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - - ymm11 = _mm256_mul_pd(ymm11, ymm7); - - ymm15 = _mm256_mul_pd(ymm15, ymm7); - - //extract a22 - ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) - - //(Row 3): FMA operations - ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); - ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); - - //(Row 3): FMA operations - ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); - ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); - - ymm10 = _mm256_mul_pd(ymm10, ymm7); - - ymm14 = _mm256_mul_pd(ymm14, ymm7); - - //extract a11 - ymm7 = _mm256_permute_pd(ymm0, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) - ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) - - //(ROW 2): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); - - ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); - - ymm9 = _mm256_mul_pd(ymm9, ymm7); - - ymm13 = _mm256_mul_pd(ymm13, ymm7); - - //extract A00 - ymm7 = _mm256_permute_pd(ymm0, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm7 = _mm256_permute2f128_pd(ymm7, ymm7, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) - - //(Row 1):FMA operations - ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); - - ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); - - ymm8 = _mm256_mul_pd(ymm8, ymm7); - - ymm12 = _mm256_mul_pd(ymm12, ymm7); - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) - } - } - } - if(i<0) - i += D_NR; - if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) - { - for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction - { - a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) - - ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha - ///GEMM for previous blocks /// - - ///load 4x4 block of b11 - ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - //multiply by alpha - ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha - ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha - ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha - ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //load 4x4 bblock of b10 - ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] - - //broadcast 1st row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) - - //braodcast 3rd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) - - //broadcast 4th row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) - - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR*cs_a; //pointer math to find next block of A for GEMM - } - - ///GEMM code end/// - - ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 - ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 - ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 - ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - a11 += cs_a; - - //2nd col - ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - - a11 += cs_a; - - //3rd col - ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - - a11 += cs_a; - - //4th col - ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] - - - ymm14 = _mm256_broadcast_sd((double const *)&ones); - - //compute reciprocals of A(i,i) and broadcast in registers - ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] - ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] - - ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] - - //extract a33 - ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - - ymm3 = _mm256_mul_pd(ymm3, ymm15); - - //extract a22 - ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) - - //(Row 3): FMA operations - ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); - ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); - - ymm2 = _mm256_mul_pd(ymm2, ymm15); - - //extract a11 - ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) - - //(ROW 2): FMA operations - ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); - - ymm1 = _mm256_mul_pd(ymm1, ymm15); - - //extract A00 - ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) - - //(Row 1):FMA operations - ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); - - ymm0 = _mm256_mul_pd(ymm0, ymm15); - - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) - - } - if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) - { - - a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) - - ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha - ///GEMM for previous blocks /// - - ///load 4x4 block of b11 - if(n_remainder == 3) - { - ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - } - if(n_remainder == 2) - { - ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - } - if(n_remainder == 1) - { - ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - } - - //multiply by alpha - ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha - ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha - ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha - ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //load 4x4 bblock of b10 - ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] - - //broadcast 1st row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) - - //braodcast 3rd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) - - //broadcast 4th row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) - - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM - } - - ///GEMM code end/// - - ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 - ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 - ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 - ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - a11 += cs_a; - - //2nd col - ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - - a11 += cs_a; - - //3rd col - ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - - a11 += cs_a; - - //4th col - ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] - - - ymm14 = _mm256_broadcast_sd((double const *)&ones); - - //compute reciprocals of A(i,i) and broadcast in registers - ymm4 = _mm256_unpacklo_pd(ymm4, ymm8); //A11[0][0] A11[1][1] A11[0][0] A11[1][1] - ymm8 = _mm256_unpacklo_pd(ymm11, ymm13); //A11[2][2] A11[3][3] A11[2][2] A11[3][3] - - ymm15 = _mm256_blend_pd(ymm4, ymm8, 0x0C); //A11[0][0] A11[1][1] A11[2][2] A11[3][3] - ymm14 = _mm256_div_pd(ymm14, ymm15); // 1/A11[0][0] 1/A11[1][1] 1/A11[2][2] 1/A11[3][3] - - //extract a33 - ymm15 = _mm256_permute_pd(ymm14, 0x0C); //(1/A11[0][0] 1/A11[0][0] 1/A11[3][3] 1/A11[3][3]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[3][3] 1/A11[3][3] 1/A11[3][3] 1/A11[3][3]) - - ymm3 = _mm256_mul_pd(ymm3, ymm15); - - //extract a22 - ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x11); //(1/A11[2][2] 1/A11[2][2] 1/A11[2][2] 1/A11[2][2]) - - //(Row 3): FMA operations - ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); - ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); - - ymm2 = _mm256_mul_pd(ymm2, ymm15); - - //extract a11 - ymm15 = _mm256_permute_pd(ymm14, 0x03); //(1/A11[1][1] 1/A11[1][1] 1/A11[2][2] 1/A11[2][2]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[1][1] 1/A11[1][1] 1/A11[1][1] 1/A11[1][1]) - - //(ROW 2): FMA operations - ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); - - ymm1 = _mm256_mul_pd(ymm1, ymm15); - - //extract A00 - ymm15 = _mm256_permute_pd(ymm14, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[2][2] 1/A11[2][2]) - ymm15 = _mm256_permute2f128_pd(ymm15, ymm15, 0x00); //(1/A11[0][0] 1/A11[0][0] 1/A11[0][0] 1/A11[0][0]) - - //(Row 1):FMA operations - ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); - - ymm0 = _mm256_mul_pd(ymm0, ymm15); - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) - _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) - } - - } - m_remainder -= 4; - i -= 4; - } - if(m_remainder) ///implementation for remainder rows - { - dtrsm_small_XAutB(L, B, AlphaVal, m_remainder, n, cs_a, cs_b); - } - return BLIS_SUCCESS; -} - -/*implements TRSM for the case XA = alpha * B - *A is lower triangular, unit-diagonal, no transpose - *dimensions: X:mxn A:nxn B: mxn - */ - -/* <---b11 <---a11 - ***************** * - *b01*b11* * * * * - ^ * * * * * ^ * * - | ***************** | ******* - | * * * * * | * * * - | * * * * * a01* * * -b10 ***************** ************* - * * * * * * * * * - * * * * * * * * * - ***************** ******************* - -*/ -static err_t bli_dtrsm_small_XAutB_unitDiag( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - dim_t D_MR = 8; //block dimension along the rows - dim_t D_NR = 4; //block dimension along the columns - - dim_t m = bli_obj_length(b); //number of rows - dim_t n = bli_obj_width(b); //number of columns - dim_t m_remainder = m % D_MR; //number of corner rows - dim_t n_remainder = n % D_NR; //number of corner columns - dim_t cs_a = bli_obj_col_stride(a); //column stride of matrix A - dim_t cs_b = bli_obj_col_stride(b); //column stride of matrix B - - if(max(m,n) > 120) - return BLIS_NOT_YET_IMPLEMENTED; - - dim_t i, j, k; //loop variablse - dim_t k_iter; //determines the number of GEMM operations to be done - dim_t cs_b_offset[2]; //pre-calculated strides - - double ones = 1.0; - - double AlphaVal = *(double *)AlphaObj->buffer; //value of Alpha - double *L = a->buffer; //pointer to matrix A - double *B = b->buffer; //pointer to matrix B - - double *a01, *a11, *b10, *b11; //pointers for GEMM and TRSM blocks - double *ptr_a01_dup; - - cs_b_offset[0] = cs_b << 1; //cs_b_offset[0] = cs_b * 2; - cs_b_offset[1] = cs_b_offset[0] + cs_b;//cs_b_offset[1] = cs_b * 3; - - //ymm scratch reginsters - __m256d ymm0, ymm1, ymm2, ymm3; - __m256d ymm4, ymm5, ymm6, ymm7; - __m256d ymm8, ymm9, ymm10, ymm11; - __m256d ymm12, ymm13, ymm14, ymm15; - __m256d ymm16; - - for(i = (m-D_MR); (i+1) > 0; i -= D_MR) //loop along 'M' direction - { - for(j = (n-D_NR); (j+1) > 0; j -= D_NR) //loop along 'N' direction - { - a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM - b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] - - a01 += cs_a; //move to next row - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] - - a01 += cs_a; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] - - a01 += cs_a; //move to next row of A01 - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] - - a01 += cs_a; //move to next row of A01 - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - //load 8x4 block of B11 - ymm8 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm12 = _mm256_loadu_pd((double const *)(b11 + D_NR)); //B11[4][0] B11[5][0] B11[6][0] B11[7][0] - ymm9 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4][1] B11[5][1] B11[6][1] B11[7][1] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4][2] B11[5][2] B11[6][2] B11[7][2] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4][3] B11[5][3] B11[6][3] B11[7][3] - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 - - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - a11 += cs_a; - - //2nd col - ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - - a11 += cs_a; - - //3rd col - ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - - a11 += cs_a; - - //4th col - ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] - - ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - - //(Row 3): FMA operations - ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); - ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); - - //(Row 3): FMA operations - ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); - ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); - - //(ROW 2): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); - - ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); - - //(Row 1):FMA operations - ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); - - ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); - - _mm256_storeu_pd((double *)b11, ymm8); //store(B11[x][0]) - _mm256_storeu_pd((double *)(b11 + D_NR), ymm12); //store(B11[x][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[x][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[x][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //(store(B11[x][2])) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //(store(B11[x][2])) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[x][3]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[x][3]) - - } - if(n_remainder) //implementation for remainder columns(when n is not multiple of D_NR) - { - - a01 = L + (j+D_NR)*cs_a +(j); //pointer to block of A to be used in GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used in GEMM - b11 = B + (i) + (j)*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of GEMM operations to be done(in blocks of 4x4) - - ymm0 = _mm256_setzero_pd(); - ymm1 = _mm256_setzero_pd(); - ymm2 = _mm256_setzero_pd(); - ymm3 = _mm256_setzero_pd(); - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //broadcast 1st row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] - - a01 += cs_a; //move to next row - - //load 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm13 = _mm256_loadu_pd((double const *)(b10 + D_NR)); //B10[4][0] B10[5][0] B10[6][0] B10[7][0] - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b + D_NR)); //B10[4][1] B10[5][1] B10[6][1] B10[7][1] - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][0]*A01[0][0] B10[5][0]*A01[0][0] B10[6][0]*A01[0][0] B10[7][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][0]*A01[0][1] B10[5][0]*A01[0][1] B10[6][0]*A01[0][1] B10[7][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][0]*A01[0][2] B10[5][0]*A01[0][2] B10[6][0]*A01[0][2] B10[7][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][0]*A01[0][3] B10[5][0]*A01[0][3] B10[6][0]*A01[0][3] B10[7][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] - - a01 += cs_a; //move to next row of A - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][1]*A01[0][0] B10[1][1]*A01[0][0] B10[2][1]*A01[0][0] B10[3][1]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][1]*A01[0][1] B10[1][1]*A01[0][1] B10[2][1]*A01[0][1] B10[3][1]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][1]*A01[0][2] B10[1][1]*A01[0][2] B10[2][1]*A01[0][2] B10[3][1]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][1]*A01[0][3] B10[1][1]*A01[0][3] B10[2][1]*A01[0][3] B10[3][1]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][1]*A01[0][0] B10[5][1]*A01[0][0] B10[6][1]*A01[0][0] B10[7][1]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][1]*A01[0][1] B10[5][1]*A01[0][1] B10[6][1]*A01[0][1] B10[7][1]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][1]*A01[0][2] B10[5][1]*A01[0][2] B10[6][1]*A01[0][2] B10[7][1]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][1]*A01[0][3] B10[5][1]*A01[0][3] B10[6][1]*A01[0][3] B10[7][1]*A01[0][3]) - - //broadcast 3rd row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] - - a01 += cs_a; //move to next row of A01 - - //load next 8x2 block of B10 - ymm12 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //(B10[0][2] B10[1][2] B10[2][2] B10[3][2]) - ymm13 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + D_NR)); //(B10[4][2] B10[5][2] B10[6][2] B10[7][2]) - ymm14 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b)); //(B10[0][3] B10[1][3] B10[2][3] B10[3][3]) - ymm15 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0] + cs_b + D_NR)); //(B10[4][3] B10[5][3] B10[6][3] B10[7][3]) - - ymm0 = _mm256_fmadd_pd(ymm8, ymm12, ymm0); //ymm0 += (B10[0][2]*A01[0][0] B10[1][2]*A01[0][0] B10[2][2]*A01[0][0] B10[3][2]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm12, ymm1); //ymm1 += (B10[0][2]*A01[0][1] B10[1][2]*A01[0][1] B10[2][2]*A01[0][1] B10[3][2]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm12, ymm2); //ymm2 += (B10[0][2]*A01[0][2] B10[1][2]*A01[0][2] B10[2][2]*A01[0][2] B10[3][2]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm12, ymm3); //ymm3 += (B10[0][2]*A01[0][3] B10[1][2]*A01[0][3] B10[2][2]*A01[0][3] B10[3][2]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm13, ymm4); //ymm4 += (B10[4][2]*A01[0][0] B10[5][2]*A01[0][0] B10[6][2]*A01[0][0] B10[7][2]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm13, ymm5); //ymm5 += (B10[4][2]*A01[0][1] B10[5][2]*A01[0][1] B10[6][2]*A01[0][1] B10[7][2]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm13, ymm6); //ymm6 += (B10[4][2]*A01[0][2] B10[5][2]*A01[0][2] B10[6][2]*A01[0][2] B10[7][2]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm13, ymm7); //ymm7 += (B10[4][2]*A01[0][3] B10[5][2]*A01[0][3] B10[6][2]*A01[0][3] B10[7][2]*A01[0][3]) - - //broadcast 4th row of A01 - ymm8 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] - ymm9 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] - ymm10 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] - ymm11 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] - - a01 += cs_a; //move to next row of A01 - - ymm0 = _mm256_fmadd_pd(ymm8, ymm14, ymm0); //ymm0 += (B10[0][3]*A01[0][0] B10[1][3]*A01[0][0] B10[2][3]*A01[0][0] B10[3][3]*A01[0][0]) - ymm1 = _mm256_fmadd_pd(ymm9, ymm14, ymm1); //ymm1 += (B10[0][3]*A01[0][1] B10[1][3]*A01[0][1] B10[2][3]*A01[0][1] B10[3][3]*A01[0][1]) - ymm2 = _mm256_fmadd_pd(ymm10, ymm14, ymm2); //ymm2 += (B10[0][3]*A01[0][2] B10[1][3]*A01[0][2] B10[2][3]*A01[0][2] B10[3][3]*A01[0][2]) - ymm3 = _mm256_fmadd_pd(ymm11, ymm14, ymm3); //ymm3 += (B10[0][3]*A01[0][3] B10[1][3]*A01[0][3] B10[2][3]*A01[0][3] B10[3][3]*A01[0][3]) - - ymm4 = _mm256_fmadd_pd(ymm8, ymm15, ymm4); //ymm4 += (B10[4][3]*A01[0][0] B10[5][3]*A01[0][0] B10[6][3]*A01[0][0] B10[7][3]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm9, ymm15, ymm5); //ymm5 += (B10[4][3]*A01[0][1] B10[5][3]*A01[0][1] B10[6][3]*A01[0][1] B10[7][3]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm10, ymm15, ymm6); //ymm6 += (B10[4][3]*A01[0][2] B10[5][3]*A01[0][2] B10[6][3]*A01[0][2] B10[7][3]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm11, ymm15, ymm7); //ymm7 += (B10[4][3]*A01[0][3] B10[5][3]*A01[0][3] B10[6][3]*A01[0][3] B10[7][3]*A01[0][3]) - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM - } - - ///GEMM code ends/// - - ymm16 = _mm256_broadcast_sd((double const *)&AlphaVal); - //load 8x4 block of B11 - if(n_remainder == 3) - { - ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - ymm9 = _mm256_loadu_pd((double const *)(b11+cs_b)); //B11[0-3][0] - ymm13 = _mm256_loadu_pd((double const *)(b11 + cs_b + D_NR)); //B11[4-7][0] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b*2)); //B11[0-3][1] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b*2 + D_NR)); //B11[4-7][1] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][2] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][2] - } - if(n_remainder == 2) - { - ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - ymm10 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0-3][0] - ymm14 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0] + D_NR)); //B11[4-7][0] - ymm11 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0-3][1] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] + D_NR)); //B11[4-7][1] - } - if(n_remainder == 1) - { - ymm8 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][1] - ymm12 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][1] - ymm9 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][2] - ymm13 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][2] - ymm10 = _mm256_broadcast_sd((double const *)&ones); //B11[0-3][3] - ymm14 = _mm256_broadcast_sd((double const *)&ones); //B11[4-7][3] - ymm11 = _mm256_loadu_pd((double const *)(b11+cs_b_offset[1])); //B11[0-3][0] - ymm15 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1] +D_NR)); //B11[4-7][0] - } - - ymm8 = _mm256_fmsub_pd(ymm8, ymm16, ymm0); //B11[0-3][0] * alpha -= ymm0 - ymm9 = _mm256_fmsub_pd(ymm9, ymm16, ymm1); //B11[4-7][0] * alpha-= ymm1 - ymm10 = _mm256_fmsub_pd(ymm10, ymm16, ymm2); //B11[0-3][1] * alpha-= ymm2 - ymm11 = _mm256_fmsub_pd(ymm11, ymm16, ymm3); //B11[4-7][1] * alpha -= ymm3 - - ymm12 = _mm256_fmsub_pd(ymm12, ymm16, ymm4); //B11[0-3][2] * alpha -= ymm4 - ymm13 = _mm256_fmsub_pd(ymm13, ymm16, ymm5); //B11[4-7][2] * alpha -= ymm5 - ymm14 = _mm256_fmsub_pd(ymm14, ymm16, ymm6); //B11[0-3][3] * alpha -= ymm6 - ymm15 = _mm256_fmsub_pd(ymm15, ymm16, ymm7); //B11[4-7][3] * alpha -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - //1st col - ymm0 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - a11 += cs_a; - - //2nd col - ymm1 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm2 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - - a11 += cs_a; - - //3rd col - ymm3 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm4 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm5 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - - a11 += cs_a; - - //4th col - ymm6 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] - - ymm2 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm5 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm6 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - - //(Row 3): FMA operations - ymm10 = _mm256_fnmadd_pd(ymm11, ymm6, ymm10); - ymm9 = _mm256_fnmadd_pd(ymm11, ymm5, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm11, ymm2, ymm8); - - //(Row 3): FMA operations - ymm14 = _mm256_fnmadd_pd(ymm15, ymm6, ymm14); - ymm13 = _mm256_fnmadd_pd(ymm15, ymm5, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm15, ymm2, ymm12); - - //(ROW 2): FMA operations - ymm9 = _mm256_fnmadd_pd(ymm10, ymm4, ymm9); - ymm8 = _mm256_fnmadd_pd(ymm10, ymm3, ymm8); - - ymm13 = _mm256_fnmadd_pd(ymm14, ymm4, ymm13); - ymm12 = _mm256_fnmadd_pd(ymm14, ymm3, ymm12); - - //(Row 1):FMA operations - ymm8 = _mm256_fnmadd_pd(ymm9, ymm1, ymm8); - - ymm12 = _mm256_fnmadd_pd(ymm13, ymm1, ymm12); - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)(b11 + cs_b), ymm9); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b + D_NR), ymm13); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14);//store(B11[4-7][2]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm10); //store(B11[0-3][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0] + D_NR), ymm14); //store(B11[4-7][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm11); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)(b11+ cs_b_offset[1]), ymm11); //store(B11[0-3][0]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1] + D_NR), ymm15); //store(B11[4-7][0]) - } - - } - } - if(i<0) - i += D_NR; - if((m & 4)) ///implementation for remainder rows(when m_remainder is a multiple of 4) - { - for(j = (n-D_NR); (j+1) > 0; j -=D_NR) //loop along n direction - { - a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) - - ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha - ///GEMM for previous blocks /// - - ///load 4x4 block of b11 - ymm0 = _mm256_loadu_pd((double const *)b11); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm1 = _mm256_loadu_pd((double const *)(b11 + cs_b)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[0])); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b_offset[1])); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - - //multiply by alpha - ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha - ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha - ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha - ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //load 4x4 bblock of b10 - ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] - - //broadcast 1st row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) - - //braodcast 3rd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) - - //broadcast 4th row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) - - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + D_NR*cs_a; //pointer math to find next block of A for GEMM - } - - ///GEMM code end/// - - ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 - ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 - ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 - ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - a11 += cs_a; - - //2nd col - ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - - a11 += cs_a; - - //3rd col - ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - - a11 += cs_a; - - //4th col - ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] - - - //(Row 3): FMA operations - ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); - ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); - - //(ROW 2): FMA operations - ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); - - //(Row 1):FMA operations - ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); - - _mm256_storeu_pd((double *)b11, ymm0); //store(B11[x][0]) - _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[1]), ymm3); //store(B11[x][3]) - - } - if(n_remainder) //implementation for remainder columns(when n is not a multiple of D_NR) - { - - a01 = L + (j+D_NR)*cs_a + (j); //pointer to block of A to be used for GEMM - a11 = L + j*cs_a + j; //pointer to block of A to be used for TRSM - b10 = B + i + (j+D_NR)*cs_b; //pointer to block of B to be used for GEMM - b11 = B + i + j*cs_b; //pointer to block of B to be used for TRSM - - k_iter = (n-j-D_NR) / D_NR; //number of times GEMM operations to be performed(in blocks of 4x4) - - ymm15 = _mm256_broadcast_sd((double const *)&AlphaVal); //register to store alpha - ///GEMM for previous blocks /// - - ///load 4x4 block of b11 - if(n_remainder == 3) - { - ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm1 = _mm256_loadu_pd((double const *)b11+ cs_b); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - } - if(n_remainder == 2) - { - ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm2 = _mm256_loadu_pd((double const *)(b11 + cs_b * 2)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - } - if(n_remainder == 1) - { - ymm0 = _mm256_broadcast_sd((double const *)&ones); //B11[0][1] B11[1][1] B11[2][1] B11[3][1] - ymm1 = _mm256_broadcast_sd((double const *)&ones); //B11[0][2] B11[1][2] B11[2][2] B11[3][2] - ymm2 = _mm256_broadcast_sd((double const *)&ones); //B11[0][3] B11[1][3] B11[2][3] B11[3][3] - ymm3 = _mm256_loadu_pd((double const *)(b11 + cs_b * 3)); //B11[0][0] B11[1][0] B11[2][0] B11[3][0] - } - - //multiply by alpha - ymm0 = _mm256_mul_pd(ymm0, ymm15); //B11[x][0] *= alpha - ymm1 = _mm256_mul_pd(ymm1, ymm15); //B11[x][1] *=alpha - ymm2 = _mm256_mul_pd(ymm2, ymm15); //B11[x][2] *= alpha - ymm3 = _mm256_mul_pd(ymm3, ymm15); //B11[x][3] *= alpha - - ymm4 = _mm256_setzero_pd(); - ymm5 = _mm256_setzero_pd(); - ymm6 = _mm256_setzero_pd(); - ymm7 = _mm256_setzero_pd(); - - ///GEMM implementation starts/// - - for(k = 0; k < k_iter; k++) //loop for number of GEMM operations - { - ptr_a01_dup = a01; - - //load 4x4 bblock of b10 - ymm8 = _mm256_loadu_pd((double const *)b10); //B10[0][0] B10[1][0] B10[2][0] B10[3][0] - ymm9 = _mm256_loadu_pd((double const *)(b10 + cs_b)); //B10[0][1] B10[1][1] B10[2][1] B10[3][1] - ymm10 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[0])); //B10[0][2] B10[1][2] B10[2][2] B10[3][2] - ymm11 = _mm256_loadu_pd((double const *)(b10 + cs_b_offset[1])); //B10[0][3] B10[1][3] B10[2][3] B10[3][3] - - //broadcast 1st row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[0][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[0][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[0][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[0][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm8, ymm4); //ymm4 += (B10[0][0]*A01[0][0] B10[1][0]*A01[0][0] B10[2][0]*A01[0][0] B10[3][0]*A01[0][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm8, ymm5); //ymm5 += (B10[0][0]*A01[0][1] B10[1][0]*A01[0][1] B10[2][0]*A01[0][1] B10[3][0]*A01[0][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm8, ymm6); //ymm6 += (B10[0][0]*A01[0][2] B10[1][0]*A01[0][2] B10[2][0]*A01[0][2] B10[3][0]*A01[0][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm8, ymm7); //ymm7 += (B10[0][0]*A01[0][3] B10[1][0]*A01[0][3] B10[2][0]*A01[0][3] B10[3][0]*A01[0][3]) - - //broadcast 2nd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[1][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[1][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[1][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[1][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm9, ymm4); //ymm4 += (B10[0][1]*A01[1][0] B10[1][1]*A01[1][0] B10[2][1]*A01[1][0] B10[3][1]*A01[1][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm9, ymm5); //ymm5 += (B10[0][1]*A01[1][1] B10[1][1]*A01[1][1] B10[2][1]*A01[1][1] B10[3][1]*A01[1][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm9, ymm6); //ymm6 += (B10[0][1]*A01[1][2] B10[1][1]*A01[1][2] B10[2][1]*A01[1][2] B10[3][1]*A01[1][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm9, ymm7); //ymm7 += (B10[0][1]*A01[1][3] B10[1][1]*A01[1][3] B10[2][1]*A01[1][3] B10[3][1]*A01[1][3]) - - //braodcast 3rd row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[2][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[2][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[2][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[2][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm10, ymm4); //ymm4 += (B10[0][2]*A01[2][0] B10[1][2]*A01[2][0] B10[2][2]*A01[2][0] B10[3][2]*A01[2][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm10, ymm5); //ymm5 += (B10[0][2]*A01[2][1] B10[1][2]*A01[2][1] B10[2][2]*A01[2][1] B10[3][2]*A01[2][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm10, ymm6); //ymm6 += (B10[0][2]*A01[2][2] B10[1][2]*A01[2][2] B10[2][2]*A01[2][2] B10[3][2]*A01[2][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm10, ymm7); //ymm7 += (B10[0][2]*A01[2][3] B10[1][2]*A01[2][3] B10[2][2]*A01[2][3] B10[3][2]*A01[2][3]) - - //broadcast 4th row of A01 - ymm12 = _mm256_broadcast_sd((double const *)(a01 + 0)); //A01[3][0] - ymm13 = _mm256_broadcast_sd((double const *)(a01 + 1)); //A01[3][1] - ymm14 = _mm256_broadcast_sd((double const *)(a01 + 2)); //A01[3][2] - ymm15 = _mm256_broadcast_sd((double const *)(a01 + 3)); //A01[3][3] - - a01 += cs_a; //move to next row of A - - ymm4 = _mm256_fmadd_pd(ymm12, ymm11, ymm4); //ymm4 += (B10[0][3]*A01[3][0] B10[1][3]*A01[3][0] B10[2][3]*A01[3][0] B10[3][3]*A01[3][0]) - ymm5 = _mm256_fmadd_pd(ymm13, ymm11, ymm5); //ymm5 += (B10[0][3]*A01[3][1] B10[1][3]*A01[3][1] B10[2][3]*A01[3][1] B10[3][3]*A01[3][1]) - ymm6 = _mm256_fmadd_pd(ymm14, ymm11, ymm6); //ymm6 += (B10[0][3]*A01[3][2] B10[1][3]*A01[3][2] B10[2][3]*A01[3][2] B10[3][3]*A01[3][2]) - ymm7 = _mm256_fmadd_pd(ymm15, ymm11, ymm7); //ymm7 += (B10[0][3]*A01[3][3] B10[1][3]*A01[3][3] B10[2][3]*A01[3][3] B10[3][3]*A01[3][3]) - - - b10 += D_NR * cs_b; //pointer math to find next block of B for GEMM - a01 = ptr_a01_dup + (D_NR * cs_a); //pointer math to find next block of A for GEMM - } - - ///GEMM code end/// - - ymm0 = _mm256_sub_pd(ymm0, ymm4); //B11[x][0] -=ymm4 - ymm1 = _mm256_sub_pd(ymm1, ymm5); //B11[x][1] -= ymm5 - ymm2 = _mm256_sub_pd(ymm2, ymm6); //B11[x][2] -= ymm6 - ymm3 = _mm256_sub_pd(ymm3, ymm7); //B11[x][3] -= ymm7 - - ///implement TRSM/// - - ///read 4x4 block of A11/// - - - //1st col - ymm4 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][0] - - a11 += cs_a; - - //2nd col - ymm5 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm8 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - - a11 += cs_a; - - //3rd col - ymm6 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm9 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm11 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - - a11 += cs_a; - - //4th col - ymm7 = _mm256_broadcast_sd((double const *)(a11+0)); //A11[0][1] - ymm10 = _mm256_broadcast_sd((double const *)(a11+1)); //A11[0][1] - ymm12 = _mm256_broadcast_sd((double const *)(a11+2)); //A11[0][1] - ymm13 = _mm256_broadcast_sd((double const *)(a11+3)); //A11[0][1] - - //(Row 3): FMA operations - ymm2 = _mm256_fnmadd_pd(ymm3, ymm12, ymm2); - ymm1 = _mm256_fnmadd_pd(ymm3, ymm10, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm3, ymm7, ymm0); - - //(ROW 2): FMA operations - ymm1 = _mm256_fnmadd_pd(ymm2, ymm9, ymm1); - ymm0 = _mm256_fnmadd_pd(ymm2, ymm6, ymm0); - - //(Row 1):FMA operations - ymm0 = _mm256_fnmadd_pd(ymm1, ymm5, ymm0); - - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)(b11 + cs_b), ymm1); //store(B11[x][1]) - _mm256_storeu_pd((double *)(b11 + cs_b_offset[0]), ymm2); //(store(B11[x][2])) - _mm256_storeu_pd((double *)(b11 + cs_b*3), ymm3); //store(B11[x][0]) - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)(b11+ cs_b * 2), ymm2); //store(B11[x][0]) - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][1]) - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)(b11 + cs_b * 3), ymm3); //store(B11[x][0]) - } - - } - m_remainder -= 4; - i -= 4; - } - if(m_remainder) - { - dtrsm_small_XAutB_unitDiag(a->buffer, b->buffer,AlphaVal, m_remainder, n, cs_a, cs_b); - } - return BLIS_SUCCESS; -} - - -/* - * AX = Alpha*B, Single precision, A: lower triangular - * This kernel implementation supports matrices A and B such that m is equal to BLI_AlXB_M_SP and n is mutiple of 8 - */ - -static err_t bli_strsm_small_AlXB ( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - obj_t alpha, beta; // gemm parameters - obj_t Ga, Gb, Gc; // for GEMM - int m = bli_obj_length(b); // number of rows of matrix B - int n = bli_obj_width(b); // number of columns of matrix B - - int lda = bli_obj_col_stride(a); // column stride of A - int ldb = bli_obj_col_stride(b); // column stride of B - - int rsa = bli_obj_row_stride(a); // row stride of A - int rsb = bli_obj_row_stride(b); // row stride of B - - int i = 0; - int j; - int blk_size = 8; - int isUnitDiag = bli_obj_has_unit_diag(a); - - float alphaVal; - float *L = a->buffer; - float *B = b->buffer; - - if (m != BLI_AlXB_M_SP || (n&7) != 0) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - if ( (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - - alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); - - /* Small _GEMM preparation code */ - bli_obj_create( BLIS_FLOAT, 1, 1, 0, 0, &alpha ); - bli_obj_create( BLIS_FLOAT, 1, 1, 0, 0, &beta ); - - /* B = B - A*B */ - bli_setsc( -(1.0), 0.0, &alpha ); - bli_setsc( (1.0), 0.0, &beta ); - - - bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, blk_size, a->buffer, rsa, lda, &Ga); - bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, n, b->buffer, rsb, ldb, &Gb); - bli_obj_create_with_attached_buffer( BLIS_FLOAT, blk_size, n, b->buffer, rsb, ldb, &Gc); - - bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Ga ); - bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gb ); - bli_obj_set_conjtrans( BLIS_NO_TRANSPOSE, &Gc ); - - //first block of trsm - Gb.buffer = (void*)(B + i); - - //trsm of first 8xn block - if (alphaVal != 1) - { - if (isUnitDiag == 0) - { - blis_strsm_microkernel_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - fp_blis_strsm_microkernel = blis_strsm_microkernel; - } - else - { - blis_strsm_microkernel_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; - } - bli_setsc( alphaVal, 0.0, &beta ); - } - else - { - if (isUnitDiag == 0) - { - blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - fp_blis_strsm_microkernel = blis_strsm_microkernel; - } - else - { - blis_strsm_microkernel_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - fp_blis_strsm_microkernel = blis_strsm_microkernel_unitDiag; - } - } - - //gemm update - for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT - { - Ga.buffer = (void*)(L + j + i*lda); - Gc.buffer = (void*)(B + j); - - bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb - } - - //trsm of remaining blocks - for (i = blk_size; i < m; i += blk_size) - { - Gb.buffer = (void*)(B + i); - - fp_blis_strsm_microkernel((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - - for (j = i + blk_size; j < m; j += blk_size) // for rows upto multiple of BLOCK_HEIGHT - { - Ga.buffer = (void*)(L + j + i*lda); - Gc.buffer = (void*)(B + j); - - bli_gemm_small(&alpha, &Ga, &Gb, &beta, &Gc, cntx, cntl ); // Gc = beta*Gc + alpha*Ga *Gb - } - - } // End of for loop - i - - return BLIS_SUCCESS; -} - - - -/* - * XA' = Alpha*B, Single precision, A: lower triangular - * This kernel implementation supports matrices A and B such that - * m and n are multiples of 8 and n is less than or equal to BLI_XAltB_N_SP - */ -static err_t bli_strsm_small_XAltB( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - int m = bli_obj_length(a); // number of rows of matrix B - int n = bli_obj_length(b); // number of columns of matrix B - - int lda = bli_obj_col_stride(a); // column stride of A - int ldb = bli_obj_col_stride(b); // column stride of B - - int rsa = bli_obj_row_stride(a); // row stride of A - int rsb = bli_obj_row_stride(b); // row stride of B - - int i = 0; - int isUnitDiag = bli_obj_has_unit_diag(a); - - float alphaVal; - float *L = a->buffer; - float *B = b->buffer; - - if ((m&7) != 0 || (n&7) != 0) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - if ( n > BLI_XAltB_N_SP || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - - alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); - - if (alphaVal != 1) - { - if (isUnitDiag == 0) - { - trsm_XAtB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - } - else - { - trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - } - } - else - { - if (isUnitDiag == 0) - { - trsm_XAtB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - } - else - { - trsm_XAtB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - } - } - return BLIS_SUCCESS; -} - -/* - * A'X = Alpha*B, Single precision, A: upper triangular - * This kernel implementation supports matrices A and B such that - * m and n are multiples of 8, m is less than or equal to BLI_AutXB_M_SP and n is less than or equal to BLI_AutXB_N_SP - */ -static err_t bli_strsm_small_AutXB( - side_t side, - obj_t* AlphaObj, - obj_t* a, - obj_t* b, - cntx_t* cntx, - cntl_t* cntl - ) -{ - int m = bli_obj_width(a); // number of rows of matrix A (since At, so width is taken) - int n = bli_obj_width(b); // number of columns of matrix B - - int lda = bli_obj_col_stride(a); // column stride of A - int ldb = bli_obj_col_stride(b); // column stride of B - - int rsa = bli_obj_row_stride(a); // row stride of A - int rsb = bli_obj_row_stride(b); // row stride of B - - int i = 0; - int isUnitDiag = bli_obj_has_unit_diag(a); - - float alphaVal; - float *L = a->buffer; - float *B = b->buffer; - - if ((m&7) != 0 || (n&7) != 0) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - if ( m > BLI_AutXB_M_SP || n > BLI_AutXB_N_SP || (m*(m + n)) > BLIS_SMALL_MATRIX_THRES_TRSM ) - { - return BLIS_NOT_YET_IMPLEMENTED; - } - - alphaVal = *((float *)bli_obj_buffer_for_const(BLIS_FLOAT, AlphaObj)); - - if (alphaVal != 1) - { - if (isUnitDiag == 0) - { - trsm_AutXB_block_allSmallSizedMatrices_alpha((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - } - else - { - trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb, alphaVal); - } - } - else - { - if (isUnitDiag == 0) - { - trsm_AutXB_block_allSmallSizedMatrices((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - } - else - { - trsm_AutXB_block_allSmallSizedMatrices_unitDiag((L + i * lda + i), (B + i), m, n, rsa, rsb, lda, ldb); - } - } - return BLIS_SUCCESS; -} - -///////////////////////////// AX=B /////////////////////////////// -static void blis_strsm_microkernel_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) -{ - float ones = 1.0; - int j; - int cs_b_offset[6]; - //int row2, row4, row6; - float *ptr_b_dup; - - //70 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_cols[8]; - __m256 mat_a_cols_rearr[36]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags; - __m256 alphaReg; - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); - alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); - //row2 = (cs_l << 1); - //row4 = (cs_l << 2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); - //row6 = row2 + row4; - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); - - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - - //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L - /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ - - //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers - //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. - //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); - mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); - mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); - mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); - mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //4rth col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //5th col - ptr_l += cs_l; - mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //6th col - ptr_l += cs_l; - mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //7th col - ptr_l += cs_l; - mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //7th col - ptr_l += cs_l; - mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - numCols_b -= 8; // blk_width = 8 - - //compute reciprocals of L(i,i) and broadcast in registers - mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); - mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); - mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); - - //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); - //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); - - //reciprocal of diagnol elements - reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); - - //Start loop for cols of B to be processed in size of blk_width - for (j = 0; j < numCols_b; j += 8) - { - ptr_b_dup = ptr_b; - - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); -#else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - //Read next set of B columns - ptr_b += (cs_b + cs_b_offset[5]); - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - - //end loop of cols - } - - //Last block trsm processing - ptr_b_dup = ptr_b; - - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); -#else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - - //end loop of cols -} - -static void blis_strsm_microkernel_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alphaVal) -{ - //float ones = 1.0; - int j; - int cs_b_offset[6]; - //int row2, row4, row6; - float *ptr_b_dup; - - //70 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_cols[8]; - __m256 mat_a_cols_rearr[36]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags; - __m256 alphaReg; - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); - alphaReg = _mm256_broadcast_ss((float const *)&alphaVal); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); - //row2 = (cs_l << 1); - //row4 = (cs_l << 2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); - //row6 = row2 + row4; - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); - - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - - //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L - /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ - - //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers - //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. - //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); - mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); - mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); - mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); - mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //4rth col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //5th col - ptr_l += cs_l; - mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //6th col - ptr_l += cs_l; - mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //7th col - ptr_l += cs_l; - mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //8th col - //ptr_l += cs_l; - //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - numCols_b -= 8; // blk_width = 8 - - //compute reciprocals of L(i,i) and broadcast in registers - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); - - //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); - //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); - //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); - //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); - //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); - - //reciprocal of diagnol elements - //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); - - //Start loop for cols of B to be processed in size of blk_width - for (j = 0; j < numCols_b; j += 8) - { - ptr_b_dup = ptr_b; - - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //extract diag a00 from a - //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); - - //extract diag a11 from a - //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - - //extract diag a44 from a - //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - - //extract diag a55 from a - //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - - //extract diag a66 from a - //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); -#else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - //Read next set of B columns - ptr_b += (cs_b + cs_b_offset[5]); - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - - //end loop of cols - } - - //Last block trsm processing - ptr_b_dup = ptr_b; - - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //extract diag a00 from a - //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); - - //extract diag a11 from a - //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - - //extract diag a44 from a - //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - - //extract diag a55 from a - //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - - //extract diag a66 from a - //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); -#else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - - //end loop of cols -} - -static void blis_strsm_microkernel_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) -{ - //float ones = 1.0; - int j; - int cs_b_offset[6]; - //int row2, row4, row6; - float *ptr_b_dup; - - //70 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_cols[8]; - __m256 mat_a_cols_rearr[36]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags; - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - //reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); - //row2 = (cs_l << 1); - //row4 = (cs_l << 2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); - //row6 = row2 + row4; - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); - - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - - //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L - /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ - - //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers - //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. - //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); - mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); - mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); - mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); - mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //4rth col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //5th col - ptr_l += cs_l; - mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //6th col - ptr_l += cs_l; - mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //7th col - ptr_l += cs_l; - mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //8th col - //ptr_l += cs_l; - //mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - numCols_b -= 8; // blk_width = 8 - - //compute reciprocals of L(i,i) and broadcast in registers - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); - - //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); - //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); - //mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); - //mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); - //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); - - //reciprocal of diagnol elements - //reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); - - //Start loop for cols of B to be processed in size of blk_width - for (j = 0; j < numCols_b; j += 8) - { - ptr_b_dup = ptr_b; - - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //extract diag a00 from a - //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - //extract diag a11 from a - //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - - //extract diag a44 from a - //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - - //extract diag a55 from a - //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - - //extract diag a66 from a - //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); -#else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - //Read next set of B columns - ptr_b += (cs_b + cs_b_offset[5]); - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - //end loop of cols - } - - //Last block trsm processing - ptr_b_dup = ptr_b; - - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //extract diag a00 from a - //mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - //mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - //extract diag a11 from a - //mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - //mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - //mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - //mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - //mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - //mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - - //extract diag a44 from a - //mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - //mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - //mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - - //extract diag a55 from a - //mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - //mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - //mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - - //extract diag a66 from a - //mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - //mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - //mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - //mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - //mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - //mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); -#else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - //end loop of cols -} - -static void blis_strsm_microkernel(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) -{ - float ones = 1.0; - int j; - int cs_b_offset[6]; - //int row2, row4, row6; - float *ptr_b_dup; - - //70 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_cols[8]; - __m256 mat_a_cols_rearr[36]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags; - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - reciprocal_diags = _mm256_broadcast_ss((float const *)&ones); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //read first set of 16x8 block of B into registers, where 16 is the blk_height and 8 is the blk_width for B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - //_mm_prefetch((char*)(ptr_l + 0), _MM_HINT_T0); - //row2 = (cs_l << 1); - //row4 = (cs_l << 2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - //_mm_prefetch((char*)(ptr_l + row2 + cs_l), _MM_HINT_T0); - //row6 = row2 + row4; - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - //_mm_prefetch((char*)(ptr_l + row4), _MM_HINT_T0); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - //_mm_prefetch((char*)(ptr_l + row4 + cs_l), _MM_HINT_T0); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - //_mm_prefetch((char*)(ptr_l + row6), _MM_HINT_T0); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - //_mm_prefetch((char*)(ptr_l + row6 + cs_l), _MM_HINT_T0); - - //reciprocal_diags = _mm256_loadu_ps((float const *)ones); - - //read first set of 16x16 block of L, where 16 is the blk_height and 16 is the blk_width for L - /*mat_a_cols[0] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[1] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[2] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[3] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[4] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[5] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[6] = _mm256_loadu_ps((float const *)ptr_l); - ptr_l += cs_l; - mat_a_cols[7] = _mm256_loadu_ps((float const *)ptr_l);*/ - - //Shuffle to rearrange/transpose 16x16 block of L into contiguous row-wise registers - //tmpRegs[0] = _mm256_castps256_ps128(mat_a_cols[0]); //zero latency, no instruction added actually. - //mat_a_cols_rearr[0] = _mm256_broadcastss_ps(tmpRegs[0]); - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_ss((float const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_ss((float const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_ss((float const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_ss((float const *)(ptr_l+3)); - mat_a_cols_rearr[10] = _mm256_broadcast_ss((float const *)(ptr_l+4)); - mat_a_cols_rearr[15] = _mm256_broadcast_ss((float const *)(ptr_l+5)); - mat_a_cols_rearr[21] = _mm256_broadcast_ss((float const *)(ptr_l+6)); - mat_a_cols_rearr[28] = _mm256_broadcast_ss((float const *)(ptr_l+7)); - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[11] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[16] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[22] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[29] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[12] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[17] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[23] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[30] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //4rth col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_cols_rearr[13] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[18] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[24] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[31] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //5th col - ptr_l += cs_l; - mat_a_cols_rearr[14] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_cols_rearr[19] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[25] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[32] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //6th col - ptr_l += cs_l; - mat_a_cols_rearr[20] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_cols_rearr[26] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[33] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //7th col - ptr_l += cs_l; - mat_a_cols_rearr[27] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_cols_rearr[34] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - //7th col - ptr_l += cs_l; - mat_a_cols_rearr[35] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - numCols_b -= 8; // blk_width = 8 - - //compute reciprocals of L(i,i) and broadcast in registers - mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); - mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_cols_rearr[14], mat_a_cols_rearr[20]); - mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_cols_rearr[27], mat_a_cols_rearr[35]); - - //mat_a_diag_inv[1] = _mm256_permute_ps(mat_a_diag_inv[1], 0x55); - //mat_a_diag_inv[3] = _mm256_permute_ps(mat_a_diag_inv[3], 0x55); - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC); - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x20); - - //reciprocal of diagnol elements - reciprocal_diags = _mm256_div_ps(reciprocal_diags, mat_a_diag_inv[0]); - - //Start loop for cols of B to be processed in size of blk_width - for (j = 0; j < numCols_b; j += 8) - { - ptr_b_dup = ptr_b; - - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); -#else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - //Read next set of B columns - ptr_b += (cs_b + cs_b_offset[5]); - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5])); - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - //end loop of cols - } - - //Last block trsm processing - ptr_b_dup = ptr_b; - - /*Shuffle to rearrange/transpose 16x8 block of B into contiguous row-wise registers*/ - - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[10], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[15], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[21], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[28], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[11], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[16], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[22], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[29], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[12], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[17], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[23], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[30], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags, 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_cols_rearr[13], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[18], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[24], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[31], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags, 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_cols_rearr[19], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[25], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[32], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags, 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_cols_rearr[26], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[33], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags, 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_cols_rearr[34], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_a_cols[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_a_cols[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_a_cols[4] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x44); - mat_a_cols[5] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0xEE); - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x44); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0xEE); -#else - mat_a_cols[6] = _mm256_shuffle_ps(mat_a_cols[0], mat_a_cols[1], 0x4E); - mat_a_cols[7] = _mm256_shuffle_ps(mat_a_cols[2], mat_a_cols[3], 0x4E); - mat_a_cols[4] = _mm256_blend_ps(mat_a_cols[0], mat_a_cols[6], 0xCC); - mat_a_cols[5] = _mm256_blend_ps(mat_a_cols[1], mat_a_cols[6], 0x33); - mat_a_cols[6] = _mm256_blend_ps(mat_a_cols[2], mat_a_cols[7], 0xCC); - mat_a_cols[7] = _mm256_blend_ps(mat_a_cols[3], mat_a_cols[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_a_cols[0] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x20); - mat_a_cols[4] = _mm256_permute2f128_ps(mat_a_cols[4], mat_a_cols[6], 0x31); - mat_a_cols[1] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x20); - mat_a_cols[5] = _mm256_permute2f128_ps(mat_a_cols[5], mat_a_cols[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_a_cols[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_a_cols[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_a_cols[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_a_cols[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_a_cols[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_a_cols[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_a_cols[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_a_cols[7]); - //end loop of cols -} -static void blis_dtrsm_microkernel_alpha(double *ptr_l, - double *ptr_b, - int m, - int n, - int rs_l, - int rs_b, - int cs_l, - int cs_b, - double alphaVal - ) -{ - int j; - int n_remainder = n%4; - int cs_b_offset[2]; - double *ptr_b_dup; - double ones = 1.0; - __m256d mat_b_col[4]; - __m256d mat_b_rearr[4]; - __m256d mat_a_cols[4]; - __m256d mat_a_cols_rearr[10]; - __m256d mat_a_diag_inv[4]; - __m256d reciprocal_diags; - __m256d alphaReg; - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - - reciprocal_diags = _mm256_broadcast_sd((double const *)&ones); - alphaReg = _mm256_broadcast_sd((double const *)&alphaVal); - - //if(m % 4 == 0) - //{ - //1st col - mat_a_cols_rearr[0] = _mm256_broadcast_sd((double const *)(ptr_l+0)); - mat_a_cols_rearr[1] = _mm256_broadcast_sd((double const *)(ptr_l+1)); - mat_a_cols_rearr[3] = _mm256_broadcast_sd((double const *)(ptr_l+2)); - mat_a_cols_rearr[6] = _mm256_broadcast_sd((double const *)(ptr_l+3)); - - //2nd col - ptr_l += cs_l; - mat_a_cols_rearr[2] = _mm256_broadcast_sd((double const *)(ptr_l + 1)); - mat_a_cols_rearr[4] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[7] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //3rd col - ptr_l += cs_l; - mat_a_cols_rearr[5] = _mm256_broadcast_sd((double const *)(ptr_l + 2)); - mat_a_cols_rearr[8] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - - //4th col - ptr_l += cs_l; - mat_a_cols_rearr[9] = _mm256_broadcast_sd((double const *)(ptr_l + 3)); - //compute reciprocals of L(i,i) and broadcast in registers - mat_a_diag_inv[0] = _mm256_unpacklo_pd(mat_a_cols_rearr[0], mat_a_cols_rearr[2]); - mat_a_diag_inv[1] = _mm256_unpacklo_pd(mat_a_cols_rearr[5], mat_a_cols_rearr[9]); - - mat_a_diag_inv[0] = _mm256_blend_pd(mat_a_diag_inv[0], mat_a_diag_inv[1], 0x0C); - reciprocal_diags = _mm256_div_pd(reciprocal_diags, mat_a_diag_inv[0]); - - for(j = 0;(j+3) < n; j += 4) - { - ptr_b_dup = ptr_b; - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - - //read first set of 4x4 block of B into registers - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - //_mm_prefetch((char*)(ptr_l + cs_l), _MM_HINT_T0); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - //_mm_prefetch((char*)(ptr_l + row2), _MM_HINT_T0); - mat_b_col[3] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[1])); - - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); - //extract a00 - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); - - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); - - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - - //Read next set of B columns - ptr_b += (cs_b+cs_b_offset[1]); - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[1]), mat_a_cols[3]); - - } - ptr_b_dup = ptr_b; - if(n_remainder == 3) - { - - //read first set of 4x4 block of B into registers - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_loadu_pd((double const *)(ptr_b + cs_b_offset[0])); - mat_b_col[3] = _mm256_broadcast_sd((double const *)&ones); - } - if(n_remainder == 2) - { - //read first set of 4x4 block of B into registers - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_loadu_pd((double const *)(ptr_b + (cs_b))); - mat_b_col[2] = _mm256_broadcast_sd((double const *)&ones); - mat_b_col[3] = _mm256_broadcast_sd((double const *)&ones); - } - if(n_remainder == 1) - { - //read first set of 4x4 block of B into registers - mat_b_col[0] = _mm256_loadu_pd((double const *)ptr_b); - mat_b_col[1] = _mm256_broadcast_sd((double const *)&ones); - mat_b_col[2] = _mm256_broadcast_sd((double const *)&ones); - mat_b_col[3] = _mm256_broadcast_sd((double const *)&ones); - } - /*Shuffle to rearrange/transpose 8x4 block of B into contiguous row-wise registers*/ - ////unpacklow//// - mat_b_rearr[1] = _mm256_unpacklo_pd(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[3] = _mm256_unpacklo_pd(mat_b_col[2], mat_b_col[3]); - //rearrange low elements - mat_b_rearr[0] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x20); - mat_b_rearr[2] = _mm256_permute2f128_pd(mat_b_rearr[1],mat_b_rearr[3],0x31); - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], alphaReg); - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], alphaReg); - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_pd(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_pd(mat_b_col[2], mat_b_col[3]); - //rearrange high elements - mat_b_rearr[1] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x20); - mat_b_rearr[3] = _mm256_permute2f128_pd(mat_b_col[0],mat_b_col[1],0x31); - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], alphaReg); - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], alphaReg); - //extract a00 - mat_a_diag_inv[0] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_pd(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_pd(mat_b_rearr[0], mat_a_diag_inv[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_pd(reciprocal_diags, 0x03); - mat_a_diag_inv[1] = _mm256_permute2f128_pd(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (3, 0) - mat_b_rearr[1] = _mm256_fnmadd_pd(mat_a_cols_rearr[1], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[3], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[6], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_pd(mat_b_rearr[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_pd(reciprocal_diags, 0x00); - mat_a_diag_inv[2] = _mm256_permute2f128_pd(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x11); - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_pd(mat_a_cols_rearr[4], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[7], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_pd(mat_b_rearr[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_pd(reciprocal_diags, 0x0C); - mat_a_diag_inv[3] = _mm256_permute2f128_pd(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x11); - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_pd(mat_a_cols_rearr[8], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_pd(mat_b_rearr[3], mat_a_diag_inv[3]); - //--> Transpose and store results of columns of B block <--// - ////unpacklow//// - mat_a_cols[1] = _mm256_unpacklo_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_a_cols[3] = _mm256_unpacklo_pd(mat_b_rearr[2], mat_b_rearr[3]); - //rearrange low elements - mat_a_cols[0] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x20); - mat_a_cols[2] = _mm256_permute2f128_pd(mat_a_cols[1],mat_a_cols[3],0x31); - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_pd(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_pd(mat_b_rearr[2], mat_b_rearr[3]); - //rearrange high elements - mat_a_cols[1] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x20); - mat_a_cols[3] = _mm256_permute2f128_pd(mat_b_rearr[0],mat_b_rearr[1],0x31); - //Store the computed B columns - if(n_remainder == 3) - { - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - _mm256_storeu_pd((double *)(ptr_b_dup + cs_b_offset[0]), mat_a_cols[2]); - } - if(n_remainder == 2) - { - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - _mm256_storeu_pd((double *)(ptr_b_dup + (cs_b)), mat_a_cols[1]); - } - if(n_remainder == 1) - { - _mm256_storeu_pd((double *)ptr_b_dup, mat_a_cols[0]); - } - - //} - -} - -#if OPT_CACHE_BLOCKING_L1 //new intrinsic kernels -static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) -{ - float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags[2]; - - reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - //read diag elems of L 16x16 block - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - - reciprocal_diags[1] = reciprocal_diags[0]; - - //pack first 8 diags together - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - - //Read next 8x8 block of A to get diag elements - i3 += cs_l_offset[6]; - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); - - //pack 8 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { -#if GEMM_ACCUM_A - i = i1 + r; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); -#endif - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - ptr_l_dup = ptr_l; - i4 = i2 + r; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - i4 = k >> 3; - ptr_l_dup += cs_l; - -#if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#endif - //end loop of cols - } - i2 += cs_b_offset[6]; - i += cs_l_offset[6]; - } - //trsm solve - - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - { - i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A -#if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); -#endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - -#if GEMM_ACCUM_A - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); -#else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); -#endif - -#if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } - } - } //numRows of A - ///////////////////loop ends ///////////////////// -} - -static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) -{ - float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags[2]; - __m256 alphaReg; - - reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); - alphaReg = _mm256_broadcast_ss((float const *)&alpha); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - //read diag elems of L 16x16 block - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - - reciprocal_diags[1] = reciprocal_diags[0]; - - //pack first 8 diags together - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); -#if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); -#endif - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - - //Read next 8x8 block of A to get diag elements - i3 += cs_l_offset[6]; - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); - - //pack 8 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { -#if GEMM_ACCUM_A - i = i1 + r; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); -#endif - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - ptr_l_dup = ptr_l; - i4 = i2 + r; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - i4 = k >> 3; - ptr_l_dup += cs_l; - -#if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#endif - //end loop of cols - } - i2 += cs_b_offset[6]; - i += cs_l_offset[6]; - } - //trsm solve - - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - { - i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A -#if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); - - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); -#endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - -#if GEMM_ACCUM_A - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); -#else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); -#endif - -#if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - - _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } - } - } //numRows of A - ///////////////////loop ends ///////////////////// -} - -static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) -{ - //float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags[2]; - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - //(Row0) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - i3 += cs_l_offset[6]; - - i = 0; - i2 = 0; - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { -#if GEMM_ACCUM_A - i = i1 + r; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); -#endif - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - ptr_l_dup = ptr_l; - i4 = i2 + r; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - i4 = k >> 3; - ptr_l_dup += cs_l; - -#if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#endif - //end loop of cols - } - i2 += cs_b_offset[6]; - i += cs_l_offset[6]; - } - //trsm solve - - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - { - i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A -#if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); -#endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - -#if GEMM_ACCUM_A - //(Row0): already done -#else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); -#endif - -#if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } - } - } //numRows of A - ///////////////////loop ends ///////////////////// -} - -static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) -{ - //float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags[2]; - __m256 alphaReg; - alphaReg = _mm256_broadcast_ss((float const *)&alpha); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - -#if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); -#endif - - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); - - //(Row0) - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - i3 += cs_l_offset[6]; - - i = 0; - i2 = 0; - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { -#if GEMM_ACCUM_A - i = i1 + r; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); -#endif - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - ptr_l_dup = ptr_l; - i4 = i2 + r; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - i4 = k >> 3; - ptr_l_dup += cs_l; - -#if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + i + 7)); - ptr_l_dup += cs_l; -#if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#endif - //end loop of cols - } - i2 += cs_b_offset[6]; - i += cs_l_offset[6]; - } - //trsm solve - - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - { - i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A -#if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i2); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i2)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i2)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i2)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i2)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i2)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i2)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i2)); - - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); -#endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - -#if GEMM_ACCUM_A - //(Row0): already done - -#else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); -#endif - -#if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + r, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+r), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + r), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + r), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + r), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + r), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + r), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + r), mat_b_rearr[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } - } - } //numRows of A - ///////////////////loop ends ///////////////////// -} -#else //rel 1.0 intrisic kernels (NOT OPT_CACHE_BLOCKING_L1) -static void trsm_XAtB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) -{ - float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[16][8]; - __m256 mat_a_cols_rearr[8]; - __m256 mat_a_blk_elems[64]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags[2]; - - reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - //read diag elems of L 16x16 block - mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); - mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); - mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); - mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); - mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); - mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - - reciprocal_diags[1] = reciprocal_diags[0]; - - //pack first 8 diags together - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - - //Read next 8x8 block of A to get diag elements - i3 += cs_l_offset[6]; - mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); - - //pack 8 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 8) - { - i = i1 + k; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - i2++; - } - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); - - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); - - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); - mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); - mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); - mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); - mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); - - // _mm256_permute2f128_ps() - - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); - mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); - mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); - mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); - mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); - mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); - mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); - mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); - - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); - mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); - mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); - mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); - mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); - mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); - mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); - mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); - - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); - mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); - mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); - mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); - mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); - mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); - mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); - mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); - - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); - mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); - mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); - mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); - mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); - mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); - mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); - mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); - - i += cs_l_offset[6]; - - - for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - i4 = k >> 3; - - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) - - //end loop of cols - } - i2 += cs_b_offset[6]; - } - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - k = 0; - for (i = 0; i < numCols_b; i+=8) - { - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - - _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } - - - } - ///////////////////loop ends ///////////////////// -} - -static void trsm_XAtB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) -{ - float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[16][8]; - __m256 mat_a_cols_rearr[8]; - __m256 mat_a_blk_elems[64]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags[2]; - __m256 alphaReg; - - reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); - alphaReg = _mm256_broadcast_ss((float const *)&alpha); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - //read diag elems of L 16x16 block - mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l); - mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); - mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); - mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); - mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); - mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - - reciprocal_diags[1] = reciprocal_diags[0]; - - //pack first 8 diags together - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); - mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); - mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); - mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); - mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); - mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); - mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); - mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_ps(mat_b_rearr[0][0], mat_a_diag_inv[0]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_ps(mat_b_rearr[1][0], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_ps(mat_b_rearr[2][0], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_ps(mat_b_rearr[3][0], mat_a_diag_inv[3]); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_col[4] = _mm256_mul_ps(mat_b_rearr[4][0], mat_a_diag_inv[4]); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_col[5] = _mm256_mul_ps(mat_b_rearr[5][0], mat_a_diag_inv[5]); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_col[6] = _mm256_mul_ps(mat_b_rearr[6][0], mat_a_diag_inv[6]); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_col[7] = _mm256_mul_ps(mat_b_rearr[7][0], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - - //Read next 8x8 block of A to get diag elements - i3 += cs_l_offset[6]; - mat_a_cols_rearr[0] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_cols_rearr[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_cols_rearr[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_cols_rearr[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_cols_rearr[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_cols_rearr[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_cols_rearr[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_cols_rearr[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); - - //pack 8 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_cols_rearr[0], mat_a_cols_rearr[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_cols_rearr[2], mat_a_cols_rearr[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_cols_rearr[4], mat_a_cols_rearr[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_cols_rearr[6], mat_a_cols_rearr[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 8) - { - i = i1 + k; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); - mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); - mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); - mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); - mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); - mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); - mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); - mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); - - i2++; - } - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); - - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); - - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); - mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); - mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); - mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); - mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); - - // _mm256_permute2f128_ps() - - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); - mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); - mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); - mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); - mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); - mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); - mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); - mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); - - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); - mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); - mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); - mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); - mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); - mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); - mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); - mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); - - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); - mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); - mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); - mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); - mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); - mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); - mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); - mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); - - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); - mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); - mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); - mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); - mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); - mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); - mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); - mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); - - i += cs_l_offset[6]; - - - for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - i4 = k >> 3; - - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) - - //end loop of cols - } - i2 += cs_b_offset[6]; - } - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - k = 0; - for (i = 0; i < numCols_b; i+=8) - { - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[k][0] = _mm256_mul_ps(mat_b_rearr[k][0], mat_a_diag_inv[0]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[k][1] = _mm256_mul_ps(mat_b_rearr[k][1], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[k][2] = _mm256_mul_ps(mat_b_rearr[k][2], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[k][3] = _mm256_mul_ps(mat_b_rearr[k][3], mat_a_diag_inv[3]); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[k][4] = _mm256_mul_ps(mat_b_rearr[k][4], mat_a_diag_inv[4]); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[k][5] = _mm256_mul_ps(mat_b_rearr[k][5], mat_a_diag_inv[5]); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[k][6] = _mm256_mul_ps(mat_b_rearr[k][6], mat_a_diag_inv[6]); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[k][7] = _mm256_mul_ps(mat_b_rearr[k][7], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - - _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); - k++; - } - - - } - ///////////////////loop ends ///////////////////// -} - -static void trsm_XAtB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) -{ - //float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[16][8]; - //__m256 mat_a_cols_rearr[8]; - __m256 mat_a_blk_elems[64]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags[2]; - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - //(Row0) - mat_b_col[0] = mat_b_rearr[0][0]; - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - i3 += cs_l_offset[6]; - - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 8) - { - i = i1 + k; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - i2++; - } - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); - - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); - - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); - mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); - mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); - mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); - mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); - - // _mm256_permute2f128_ps() - - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); - mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); - mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); - mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); - mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); - mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); - mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); - mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); - - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); - mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); - mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); - mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); - mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); - mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); - mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); - mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); - - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); - mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); - mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); - mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); - mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); - mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); - mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); - mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); - - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); - mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); - mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); - mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); - mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); - mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); - mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); - mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); - - i += cs_l_offset[6]; - - for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - i4 = k >> 3; - - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) - - //end loop of cols - } - i2 += cs_b_offset[6]; - } - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - k = 0; - for (i = 0; i < numCols_b; i+=8) - { - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A - - //(Row0): already done - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - - _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } - - - } - ///////////////////loop ends ///////////////////// -} - -static void trsm_XAtB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) -{ - //float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[16][8]; - //__m256 mat_a_cols_rearr[8]; - __m256 mat_a_blk_elems[64]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags[2]; - __m256 alphaReg; - alphaReg = _mm256_broadcast_ss((float const *)&alpha); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); - - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7][0] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - mat_b_rearr[0][0] = _mm256_mul_ps(mat_b_rearr[0][0], alphaReg); - mat_b_rearr[1][0] = _mm256_mul_ps(mat_b_rearr[1][0], alphaReg); - mat_b_rearr[2][0] = _mm256_mul_ps(mat_b_rearr[2][0], alphaReg); - mat_b_rearr[3][0] = _mm256_mul_ps(mat_b_rearr[3][0], alphaReg); - mat_b_rearr[4][0] = _mm256_mul_ps(mat_b_rearr[4][0], alphaReg); - mat_b_rearr[5][0] = _mm256_mul_ps(mat_b_rearr[5][0], alphaReg); - mat_b_rearr[6][0] = _mm256_mul_ps(mat_b_rearr[6][0], alphaReg); - mat_b_rearr[7][0] = _mm256_mul_ps(mat_b_rearr[7][0], alphaReg); - - //(Row0) - mat_b_col[0] = mat_b_rearr[0][0]; - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[1][0]);//d = c - (a*b) - mat_b_rearr[2][0] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[7][0]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[2][0]);//d = c - (a*b) - mat_b_rearr[3][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[7][0]);//d = c - (a*b) - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[2], mat_b_rearr[3][0]);//d = c - (a*b) - mat_b_rearr[4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[2], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[2], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[7][0]);//d = c - (a*b) - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[3], mat_b_rearr[4][0]);//d = c - (a*b) - mat_b_rearr[5][0] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[3], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[3], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[3], mat_b_rearr[7][0]);//d = c - (a*b) - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[4], mat_b_rearr[5][0]);//d = c - (a*b) - mat_b_rearr[6][0] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[4], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[4], mat_b_rearr[7][0]);//d = c - (a*b) - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[5], mat_b_rearr[6][0]);//d = c - (a*b) - mat_b_rearr[7][0] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[5], mat_b_rearr[7][0]);//d = c - (a*b) - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[6], mat_b_rearr[7][0]);//d = c - (a*b) - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_col[7]); - - //i += cs_b_offset[6]; - //ptr_b_dup += cs_b_offset[6]; - i += 8; - ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += 8; - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += cs_b_offset[6]; - i1 += cs_b_offset[6]; - i3 += cs_l_offset[6]; - - i = 0; - i2 = 0; - for (k = 0; k < numCols_b; k += 8) - { - i = i1 + k; - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[i2][0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[i2][1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[i2][2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[i2][3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[i2][4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[i2][5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[i2][6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[i2][7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - mat_b_rearr[i2][0] = _mm256_mul_ps(mat_b_rearr[i2][0], alphaReg); - mat_b_rearr[i2][1] = _mm256_mul_ps(mat_b_rearr[i2][1], alphaReg); - mat_b_rearr[i2][2] = _mm256_mul_ps(mat_b_rearr[i2][2], alphaReg); - mat_b_rearr[i2][3] = _mm256_mul_ps(mat_b_rearr[i2][3], alphaReg); - mat_b_rearr[i2][4] = _mm256_mul_ps(mat_b_rearr[i2][4], alphaReg); - mat_b_rearr[i2][5] = _mm256_mul_ps(mat_b_rearr[i2][5], alphaReg); - mat_b_rearr[i2][6] = _mm256_mul_ps(mat_b_rearr[i2][6], alphaReg); - mat_b_rearr[i2][7] = _mm256_mul_ps(mat_b_rearr[i2][7], alphaReg); - - i2++; - } - - i = 0; - i2 = 0; - for (l = 0; l < j; l += 8) // move across m - { - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 1)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 2)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 3)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 4)); - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 5)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 6)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + i + 7)); - - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 1)); - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 2)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 3)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 4)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 5)); - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 6)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + i + 7)); - - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i)); - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 1)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 2)); - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 3)); - mat_a_blk_elems[28] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 4)); - mat_a_blk_elems[29] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 5)); - mat_a_blk_elems[30] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 6)); - mat_a_blk_elems[31] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + i + 7)); - - // _mm256_permute2f128_ps() - - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[32] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i)); - mat_a_blk_elems[33] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 1)); - mat_a_blk_elems[34] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 2)); - mat_a_blk_elems[35] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 3)); - mat_a_blk_elems[36] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 4)); - mat_a_blk_elems[37] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 5)); - mat_a_blk_elems[38] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 6)); - mat_a_blk_elems[39] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + i + 7)); - - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[40] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i)); - mat_a_blk_elems[41] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 1)); - mat_a_blk_elems[42] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 2)); - mat_a_blk_elems[43] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 3)); - mat_a_blk_elems[44] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 4)); - mat_a_blk_elems[45] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 5)); - mat_a_blk_elems[46] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 6)); - mat_a_blk_elems[47] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + i + 7)); - - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[48] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i)); - mat_a_blk_elems[49] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 1)); - mat_a_blk_elems[50] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 2)); - mat_a_blk_elems[51] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 3)); - mat_a_blk_elems[52] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 4)); - mat_a_blk_elems[53] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 5)); - mat_a_blk_elems[54] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 6)); - mat_a_blk_elems[55] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + i + 7)); - - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[56] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i)); - mat_a_blk_elems[57] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 1)); - mat_a_blk_elems[58] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 2)); - mat_a_blk_elems[59] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 3)); - mat_a_blk_elems[60] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 4)); - mat_a_blk_elems[61] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 5)); - mat_a_blk_elems[62] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 6)); - mat_a_blk_elems[63] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5] + i + 7)); - - i += cs_l_offset[6]; - - for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - { - /////////////////// Partial Lower 8x8 block trsm of B - - i4 = i2 + k; - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - i4 = k >> 3; - - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_col[1], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_col[1], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_col[1], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_col[1], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_col[1], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_col[1], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_col[1], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_col[1], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_col[2], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_col[2], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_col[2], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_col[2], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_col[2], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_col[2], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_col[2], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_col[2], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_col[3], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_col[3], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_col[3], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_col[3], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[28], mat_b_col[3], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[29], mat_b_col[3], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[30], mat_b_col[3], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[31], mat_b_col[3], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[32], mat_b_col[4], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[33], mat_b_col[4], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[34], mat_b_col[4], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[35], mat_b_col[4], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[36], mat_b_col[4], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[37], mat_b_col[4], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[38], mat_b_col[4], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[39], mat_b_col[4], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[40], mat_b_col[5], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[41], mat_b_col[5], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[42], mat_b_col[5], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[43], mat_b_col[5], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[44], mat_b_col[5], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[45], mat_b_col[5], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[46], mat_b_col[5], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[47], mat_b_col[5], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[48], mat_b_col[6], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[49], mat_b_col[6], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[50], mat_b_col[6], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[51], mat_b_col[6], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[52], mat_b_col[6], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[53], mat_b_col[6], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[54], mat_b_col[6], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[55], mat_b_col[6], mat_b_rearr[i4][7]);//d = c - (a*b) - - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[i4][0] = _mm256_fnmadd_ps(mat_a_blk_elems[56], mat_b_col[7], mat_b_rearr[i4][0]);//d = c - (a*b) - mat_b_rearr[i4][1] = _mm256_fnmadd_ps(mat_a_blk_elems[57], mat_b_col[7], mat_b_rearr[i4][1]);//d = c - (a*b) - mat_b_rearr[i4][2] = _mm256_fnmadd_ps(mat_a_blk_elems[58], mat_b_col[7], mat_b_rearr[i4][2]);//d = c - (a*b) - mat_b_rearr[i4][3] = _mm256_fnmadd_ps(mat_a_blk_elems[59], mat_b_col[7], mat_b_rearr[i4][3]);//d = c - (a*b) - mat_b_rearr[i4][4] = _mm256_fnmadd_ps(mat_a_blk_elems[60], mat_b_col[7], mat_b_rearr[i4][4]);//d = c - (a*b) - mat_b_rearr[i4][5] = _mm256_fnmadd_ps(mat_a_blk_elems[61], mat_b_col[7], mat_b_rearr[i4][5]);//d = c - (a*b) - mat_b_rearr[i4][6] = _mm256_fnmadd_ps(mat_a_blk_elems[62], mat_b_col[7], mat_b_rearr[i4][6]);//d = c - (a*b) - mat_b_rearr[i4][7] = _mm256_fnmadd_ps(mat_a_blk_elems[63], mat_b_col[7], mat_b_rearr[i4][7]);//d = c - (a*b) - - //end loop of cols - } - i2 += cs_b_offset[6]; - } - - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + i + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + i + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + i + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + i + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + i + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + i + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - i += cs_l; - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + i + 7)); - - k = 0; - for (i = 0; i < numCols_b; i+=8) - { - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A - - //(Row0): already done - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[k][1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[k][0], mat_b_rearr[k][1]);//d = c - (a*b) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[k][0], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[k][0], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[k][0], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[k][0], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[k][0], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[k][0], mat_b_rearr[k][7]);//d = c - (a*b) - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[k][2] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_rearr[k][1], mat_b_rearr[k][2]);//d = c - (a*b) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[8], mat_b_rearr[k][1], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[9], mat_b_rearr[k][1], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[10], mat_b_rearr[k][1], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[11], mat_b_rearr[k][1], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[12], mat_b_rearr[k][1], mat_b_rearr[k][7]);//d = c - (a*b) - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[k][3] = _mm256_fnmadd_ps(mat_a_blk_elems[13], mat_b_rearr[k][2], mat_b_rearr[k][3]);//d = c - (a*b) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[14], mat_b_rearr[k][2], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[15], mat_b_rearr[k][2], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[16], mat_b_rearr[k][2], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[17], mat_b_rearr[k][2], mat_b_rearr[k][7]);//d = c - (a*b) - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[k][4] = _mm256_fnmadd_ps(mat_a_blk_elems[18], mat_b_rearr[k][3], mat_b_rearr[k][4]);//d = c - (a*b) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[19], mat_b_rearr[k][3], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[20], mat_b_rearr[k][3], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[21], mat_b_rearr[k][3], mat_b_rearr[k][7]);//d = c - (a*b) - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[k][5] = _mm256_fnmadd_ps(mat_a_blk_elems[22], mat_b_rearr[k][4], mat_b_rearr[k][5]);//d = c - (a*b) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[23], mat_b_rearr[k][4], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[24], mat_b_rearr[k][4], mat_b_rearr[k][7]);//d = c - (a*b) - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[k][6] = _mm256_fnmadd_ps(mat_a_blk_elems[25], mat_b_rearr[k][5], mat_b_rearr[k][6]);//d = c - (a*b) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[26], mat_b_rearr[k][5], mat_b_rearr[k][7]);//d = c - (a*b) - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[k][7] = _mm256_fnmadd_ps(mat_a_blk_elems[27], mat_b_rearr[k][6], mat_b_rearr[k][7]);//d = c - (a*b) - - //////////////////////////////////////////////////////////////////////////////// - - //Store the computed B columns - - _mm256_storeu_ps((float *)ptr_b_dup + i, mat_b_rearr[k][0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b) + i), mat_b_rearr[k][1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i), mat_b_rearr[k][2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i), mat_b_rearr[k][3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i), mat_b_rearr[k][4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i), mat_b_rearr[k][5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i), mat_b_rearr[k][6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i), mat_b_rearr[k][7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - } - - - } - ///////////////////loop ends ///////////////////// -} -#endif //OPT_CACHE_BLOCKING_L1 - -//////////////////////////// AutX=B /////////////////////// -static void trsm_AutXB_block_allSmallSizedMatrices(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) -{ - float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags[2]; - - reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - //read diag elems of L 16x16 block - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - - reciprocal_diags[1] = reciprocal_diags[0]; - - //pack first 8 diags together - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); -#if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); -#endif - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ - - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); - - i += cs_b_offset[6]; - ptr_b_dup += cs_b_offset[6]; - //i += 8; - //ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += cs_l_offset[6]; - - //Read next 8x8 block of A to get diag elements - i3 += 8; - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); - - //pack 8 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += 8; - i1 += 8; - i = i1; - i2 = 0; - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { -#if GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ -#endif - //i = 0; - ptr_l_dup = ptr_l; - i4 = i2; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - //{ - /////////////////// Partial Lower 8x8 block trsm of B - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); - mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); -#else - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); - mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); - mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); - mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); - mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); - /* transpose steps end */ - - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i4 = k >> 3; - ptr_l_dup++; - -#if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#endif - //end loop of cols - //} - //i2 += cs_b_offset[6]; - i4 += 8; - } - //trsm solve - - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - //{ - //i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A -#if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ -#endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i += cs_l; - -#if GEMM_ACCUM_A - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); -#else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); -#endif - -#if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); - //i += cs_l; - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); - //i += cs_l; - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); - //i += cs_l; - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); - //i += cs_l; - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); - //i += cs_l; - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - //} - i += cs_b_offset[6]; - i2 += cs_b_offset[6]; - } - } //numRows of A - ///////////////////loop ends ///////////////////// -} - -static void trsm_AutXB_block_allSmallSizedMatrices_alpha(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) -{ - float ones = 1.0; - int i, i1, i2, i3, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - __m256 mat_a_diag_inv[8]; - __m256 reciprocal_diags[2]; - __m256 alphaReg; - - reciprocal_diags[0] = _mm256_broadcast_ss((float const *)(&ones)); - alphaReg = _mm256_broadcast_ss((float const *)&alpha); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - //read diag elems of L 16x16 block - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + cs_l_offset[5]); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - - reciprocal_diags[1] = reciprocal_diags[0]; - - //pack first 8 diags together - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); -#if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); -#endif - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv[0] = _mm256_unpacklo_ps(mat_a_diag_inv[0], mat_a_diag_inv[0]); - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ - - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); - - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], mat_a_diag_inv[0]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], mat_a_diag_inv[1]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], mat_a_diag_inv[2]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], mat_a_diag_inv[3]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], mat_a_diag_inv[4]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], mat_a_diag_inv[5]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], mat_a_diag_inv[6]); - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); - - i += cs_b_offset[6]; - ptr_b_dup += cs_b_offset[6]; - //i += 8; - //ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i3 = 0; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += cs_l_offset[6]; - - //Read next 8x8 block of A to get diag elements - i3 += 8; - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_l + i3); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[0]); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[1]); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[2]); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[3]); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[4]); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)ptr_l + i3 + cs_l_offset[5]); - - //pack 8 diags of A together - reciprocal_diags[0] = reciprocal_diags[1]; - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xAA);//diag 0,1 - mat_a_diag_inv[1] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xAA);//diag 2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_blk_elems[4], mat_a_blk_elems[5], 0xAA);//diag 4,5 - mat_a_diag_inv[3] = _mm256_blend_ps(mat_a_blk_elems[6], mat_a_blk_elems[7], 0xAA);//diag 6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[1], 0xCC);//diag 0,1,2,3 - mat_a_diag_inv[2] = _mm256_blend_ps(mat_a_diag_inv[2], mat_a_diag_inv[3], 0xCC);//diag 4,5,6,7 - mat_a_diag_inv[0] = _mm256_blend_ps(mat_a_diag_inv[0], mat_a_diag_inv[2], 0xF0);//diag 0,1,2,3,4,5,6,7 - - //reciprocal of diagnal elements of A :- 0,1,2,3,4,5,6,7 - reciprocal_diags[0] = _mm256_div_ps(reciprocal_diags[0], mat_a_diag_inv[0]); - - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += 8; - i1 += 8; - i = i1; - i2 = 0; - - //extract diag a00 from a - mat_a_diag_inv[0] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[0] = _mm256_permute2f128_ps(mat_a_diag_inv[0], mat_a_diag_inv[0], 0x00); - //mat_a_diag_inv2[0] = _mm256_unpacklo_ps(mat_a_diag_inv2[0], mat_a_diag_inv2[0]); - - //extract diag a11 from a - mat_a_diag_inv[1] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[1] = _mm256_permute2f128_ps(mat_a_diag_inv[1], mat_a_diag_inv[1], 0x00); - //mat_a_diag_inv[1] = _mm256_unpacklo_ps(mat_a_diag_inv[1], mat_a_diag_inv[1]); - - //extract diag a22 from a - mat_a_diag_inv[2] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[2] = _mm256_permute2f128_ps(mat_a_diag_inv[2], mat_a_diag_inv[2], 0x00); - //mat_a_diag_inv[2] = _mm256_unpacklo_ps(mat_a_diag_inv[2], mat_a_diag_inv[2]); - - //extract diag a33 from a - mat_a_diag_inv[3] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[3] = _mm256_permute2f128_ps(mat_a_diag_inv[3], mat_a_diag_inv[3], 0x00); - //mat_a_diag_inv[3] = _mm256_unpacklo_ps(mat_a_diag_inv[3], mat_a_diag_inv[3]); - - //extract diag a44 from a - mat_a_diag_inv[4] = _mm256_permute_ps(reciprocal_diags[0], 0x00); - mat_a_diag_inv[4] = _mm256_permute2f128_ps(mat_a_diag_inv[4], mat_a_diag_inv[4], 0x11); - //mat_a_diag_inv[4] = _mm256_unpacklo_ps(mat_a_diag_inv[4], mat_a_diag_inv[4]); - - //extract diag a55 from a - mat_a_diag_inv[5] = _mm256_permute_ps(reciprocal_diags[0], 0x55); - mat_a_diag_inv[5] = _mm256_permute2f128_ps(mat_a_diag_inv[5], mat_a_diag_inv[5], 0x11); - //mat_a_diag_inv[5] = _mm256_unpacklo_ps(mat_a_diag_inv[5], mat_a_diag_inv[5]); - - //extract diag a66 from a - mat_a_diag_inv[6] = _mm256_permute_ps(reciprocal_diags[0], 0xAA); - mat_a_diag_inv[6] = _mm256_permute2f128_ps(mat_a_diag_inv[6], mat_a_diag_inv[6], 0x11); - //mat_a_diag_inv[6] = _mm256_unpacklo_ps(mat_a_diag_inv[6], mat_a_diag_inv[6]); - - //extract diag a77 from a - mat_a_diag_inv[7] = _mm256_permute_ps(reciprocal_diags[0], 0xFF); - mat_a_diag_inv[7] = _mm256_permute2f128_ps(mat_a_diag_inv[7], mat_a_diag_inv[7], 0x11); - //mat_a_diag_inv[7] = _mm256_unpacklo_ps(mat_a_diag_inv[7], mat_a_diag_inv[7]); - - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { -#if GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ - - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); -#endif - - //i = 0; - ptr_l_dup = ptr_l; - i4 = i2; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - //{ - /////////////////// Partial Lower 8x8 block trsm of B - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); - mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); -#else - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); - mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); - mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); - mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); - mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); - /* transpose steps end */ - - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i4 = k >> 3; - ptr_l_dup++; - -#if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#endif - //end loop of cols - //} - //i2 += cs_b_offset[6]; - i4 += 8; - } - //trsm solve - - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - //{ - //i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A -#if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ - - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); -#endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i += cs_l; - -#if GEMM_ACCUM_A - //(Row0): Perform mul operation of reciprocal of L(0,0) element with 1st row elements of B - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); -#else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], mat_a_diag_inv[0]); -#endif - -#if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); - //i += cs_l; - - //Perform mul operation of reciprocal of L(1,1) element with 2nd row elements of B - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], mat_a_diag_inv[1]); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); - //i += cs_l; - - //Perform mul operation of reciprocal of L(2, 2) element with 3rd row elements of B - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], mat_a_diag_inv[2]); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); - //i += cs_l; - - //Perform mul operation of reciprocal of L(3, 3) element with 4rth row elements of B - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], mat_a_diag_inv[3]); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); - //i += cs_l; - - //Perform mul operation of reciprocal of L(4, 4) element with 4rth row elements of B - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], mat_a_diag_inv[4]); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); - //i += cs_l; - - //Perform mul operation of reciprocal of L(5, 5) element with 5th row elements of B - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], mat_a_diag_inv[5]); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); - - //Perform mul operation of reciprocal of L(6, 6) element with 6th row elements of B - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], mat_a_diag_inv[6]); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - //Perform mul operation of reciprocal of L(7, 7) element with 7th row elements of B - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], mat_a_diag_inv[7]); - - //////////////////////////////////////////////////////////////////////////////// - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - //} - i += cs_b_offset[6]; - i2 += cs_b_offset[6]; - } - } //numRows of A - ///////////////////loop ends ///////////////////// -} - -static void trsm_AutXB_block_allSmallSizedMatrices_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b) -{ - //float ones = 1.0; - int i, i1, i2, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags[2]; - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - -#if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); -#endif - - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ - - - //(Row0) - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - - - - //////////////////////////////////////////////////////////////////////////////// - - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); - - i += cs_b_offset[6]; - ptr_b_dup += cs_b_offset[6]; - //i += 8; - //ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += cs_l_offset[6]; - - - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += 8; - i1 += 8; - i = i1; - i2 = 0; - - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { -#if GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ -#endif - - //i = 0; - ptr_l_dup = ptr_l; - i4 = i2; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - //{ - /////////////////// Partial Lower 8x8 block trsm of B - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); - mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); -#else - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); - mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); - mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); - mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); - mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); - /* transpose steps end */ - - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i4 = k >> 3; - ptr_l_dup++; - -#if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#endif - //end loop of cols - //} - //i2 += cs_b_offset[6]; - i4 += 8; - } - //trsm solve - - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - //{ - //i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A -#if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ -#endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i += cs_l; - -#if GEMM_ACCUM_A - //(Row0): already done - -#else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); -#endif - -#if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); - //i += cs_l; - - - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); - //i += cs_l; - - - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); - //i += cs_l; - - - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); - //i += cs_l; - - - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); - //i += cs_l; - - - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); - - - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - - - //////////////////////////////////////////////////////////////////////////////// - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - //} - i += cs_b_offset[6]; - i2 += cs_b_offset[6]; - } - } //numRows of A - ///////////////////loop ends ///////////////////// -} - -static void trsm_AutXB_block_allSmallSizedMatrices_alpha_unitDiag(float *ptr_l, float *ptr_b, int numRows_lb, int numCols_b, int rs_l, int rs_b, int cs_l, int cs_b, float alpha) -{ - //float ones = 1.0; - int i, i1, i2, i4, j, k, l, r; - int cs_b_offset[7]; - int cs_l_offset[7]; - float *ptr_b_dup, *ptr_l_dup; - - //57 number of ymm(256 bits) registers used - __m256 mat_b_col[8]; - __m256 mat_b_rearr[8]; - __m256 mat_a_blk_elems[8]; - //__m256 mat_a_diag_inv[8]; - //__m256 reciprocal_diags[2]; - __m256 alphaReg; - alphaReg = _mm256_broadcast_ss((float const *)&alpha); - - // ---> considering that the matrix size is multiple of 16 rows and 8 cols <--- // - - //L matrix offsets - cs_l_offset[0] = (cs_l << 1); - cs_l_offset[1] = cs_l + cs_l_offset[0]; - cs_l_offset[2] = (cs_l << 2); - cs_l_offset[3] = cs_l + cs_l_offset[2]; - cs_l_offset[4] = cs_l_offset[0] + cs_l_offset[2]; - cs_l_offset[5] = cs_l + cs_l_offset[4]; - cs_l_offset[6] = (cs_l_offset[5] + cs_l); - - cs_b_offset[0] = (cs_b << 1); - cs_b_offset[1] = cs_b + cs_b_offset[0]; - cs_b_offset[2] = (cs_b << 2); - cs_b_offset[3] = cs_b + cs_b_offset[2]; - cs_b_offset[4] = cs_b_offset[0] + cs_b_offset[2]; - cs_b_offset[5] = cs_b + cs_b_offset[4]; - cs_b_offset[6] = (cs_b_offset[5] + cs_b); - -#if 0 - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3)); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 4)); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 5)); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 6)); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + 7)); - - //Broadcast A21 to A71 to registers - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 2)); - mat_a_blk_elems[8] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 3)); - mat_a_blk_elems[9] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 4)); - mat_a_blk_elems[10] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 5)); - mat_a_blk_elems[11] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 6)); - mat_a_blk_elems[12] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l + 7)); - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[13] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 3)); - mat_a_blk_elems[14] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 4)); - mat_a_blk_elems[15] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 5)); - mat_a_blk_elems[16] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 6)); - mat_a_blk_elems[17] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0] + 7)); - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[18] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 4)); - mat_a_blk_elems[19] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 5)); - mat_a_blk_elems[20] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 6)); - mat_a_blk_elems[21] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1] + 7)); - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[22] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 5)); - mat_a_blk_elems[23] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 6)); - mat_a_blk_elems[24] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2] + 7)); - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[25] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 6)); - mat_a_blk_elems[26] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3] + 7)); - - //Broadcast A76 to register - mat_a_blk_elems[27] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4] + 7)); -#endif - - - /***************** first set of 8 rows of B processing starts *****************/ - ptr_b_dup = ptr_b; - i = 0; - for (j = 0; j < numCols_b; j += 8) - { - /////////////////// Complete Upper 8x8 block trsm of B :- upper 8x8 block of B with upper 8x8 block of A - //read 8x8 block of B into registers - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ - - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); - - //(Row0) - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l + cs_l_offset[5])); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_col[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_col[1]);//d = c - (a*b) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_col[7]);//d = c - (a*b) - - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l + 1 + cs_l_offset[5])); - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_col[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_col[2]);//d = c - (a*b) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_col[7]);//d = c - (a*b) - - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l + 2 + cs_l_offset[5])); - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_col[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_col[3]);//d = c - (a*b) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_col[7]);//d = c - (a*b) - - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l + 3 + cs_l_offset[5])); - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_col[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_col[4]);//d = c - (a*b) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_col[7]);//d = c - (a*b) - - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l + 4 + cs_l_offset[5])); - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_col[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_col[5]);//d = c - (a*b) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_col[7]);//d = c - (a*b) - - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l + 5 + cs_l_offset[5])); - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_col[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_col[6]);//d = c - (a*b) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_col[7]);//d = c - (a*b) - - - - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l + 6 + cs_l_offset[5])); - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_col[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_col[7]);//d = c - (a*b) - - - - //////////////////////////////////////////////////////////////////////////////// - - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup, mat_b_rearr[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)), mat_b_rearr[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0]), mat_b_rearr[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1]), mat_b_rearr[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2]), mat_b_rearr[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3]), mat_b_rearr[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4]), mat_b_rearr[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5]), mat_b_rearr[7]); - - i += cs_b_offset[6]; - ptr_b_dup += cs_b_offset[6]; - //i += 8; - //ptr_b_dup += 8; - } - - //c = 0; - /***************** first set of 8 cols of B processing done *****************/ - ptr_b_dup = ptr_b; - i1 = 0; - //Start loop for cols of B to be processed in size of blk_width - for (j = 8; j < numRows_lb; j += 8)//m :- 8x8 block row - { - ptr_l += cs_l_offset[6]; - - - //ptr_b += j; - //ptr_b_dup += 8; - ptr_b_dup += 8; - i1 += 8; - i = i1; - i2 = 0; - - for (r = 0; r < numCols_b; r += GEMM_BLK_V1) - { -#if GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_col[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_col[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_col[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_col[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_col[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_col[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_col[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_col[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - /* transpose steps start */ - ////unpacklow//// - mat_b_rearr[0] = _mm256_unpacklo_ps(mat_b_col[0], mat_b_col[1]); - mat_b_rearr[1] = _mm256_unpacklo_ps(mat_b_col[2], mat_b_col[3]); - mat_b_rearr[2] = _mm256_unpacklo_ps(mat_b_col[4], mat_b_col[5]); - mat_b_rearr[3] = _mm256_unpacklo_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_rearr[0] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_rearr[4] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_rearr[1] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_rearr[5] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - - ////unpackhigh//// - mat_b_col[0] = _mm256_unpackhi_ps(mat_b_col[0], mat_b_col[1]); - mat_b_col[1] = _mm256_unpackhi_ps(mat_b_col[2], mat_b_col[3]); - mat_b_col[2] = _mm256_unpackhi_ps(mat_b_col[4], mat_b_col[5]); - mat_b_col[3] = _mm256_unpackhi_ps(mat_b_col[6], mat_b_col[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_rearr[2] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_rearr[6] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_rearr[3] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_rearr[7] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - /* transpose steps end */ - - mat_b_rearr[0] = _mm256_mul_ps(mat_b_rearr[0], alphaReg); - mat_b_rearr[1] = _mm256_mul_ps(mat_b_rearr[1], alphaReg); - mat_b_rearr[2] = _mm256_mul_ps(mat_b_rearr[2], alphaReg); - mat_b_rearr[3] = _mm256_mul_ps(mat_b_rearr[3], alphaReg); - mat_b_rearr[4] = _mm256_mul_ps(mat_b_rearr[4], alphaReg); - mat_b_rearr[5] = _mm256_mul_ps(mat_b_rearr[5], alphaReg); - mat_b_rearr[6] = _mm256_mul_ps(mat_b_rearr[6], alphaReg); - mat_b_rearr[7] = _mm256_mul_ps(mat_b_rearr[7], alphaReg); -#endif - - //i = 0; - ptr_l_dup = ptr_l; - i4 = i2; - for (l = 0; l < j; l += 8) // move across m - { - //for (k = 0; k < numCols_b; k += 8) // move across n for the same value of l (index of m) - //{ - /////////////////// Partial Lower 8x8 block trsm of B - //Read current 8 cols of B columns from specified 8x8 current-block of B - mat_a_blk_elems[0] = _mm256_loadu_ps((float const *)ptr_b + i4); - mat_a_blk_elems[1] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b)); - mat_a_blk_elems[2] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[0])); - mat_a_blk_elems[3] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[1])); - mat_a_blk_elems[4] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[2])); - mat_a_blk_elems[5] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[3])); - mat_a_blk_elems[6] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[4])); - mat_a_blk_elems[7] = _mm256_loadu_ps((float const *)(ptr_b + i4 + cs_b_offset[5])); - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_a_blk_elems[0] = _mm256_unpackhi_ps(mat_a_blk_elems[0], mat_a_blk_elems[1]); - mat_a_blk_elems[1] = _mm256_unpackhi_ps(mat_a_blk_elems[2], mat_a_blk_elems[3]); - mat_a_blk_elems[2] = _mm256_unpackhi_ps(mat_a_blk_elems[4], mat_a_blk_elems[5]); - mat_a_blk_elems[3] = _mm256_unpackhi_ps(mat_a_blk_elems[6], mat_a_blk_elems[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_a_blk_elems[4] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x44); - mat_a_blk_elems[5] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0xEE); - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x44); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0xEE); -#else - mat_a_blk_elems[6] = _mm256_shuffle_ps(mat_a_blk_elems[0], mat_a_blk_elems[1], 0x4E); - mat_a_blk_elems[7] = _mm256_shuffle_ps(mat_a_blk_elems[2], mat_a_blk_elems[3], 0x4E); - mat_a_blk_elems[4] = _mm256_blend_ps(mat_a_blk_elems[0], mat_a_blk_elems[6], 0xCC); - mat_a_blk_elems[5] = _mm256_blend_ps(mat_a_blk_elems[1], mat_a_blk_elems[6], 0x33); - mat_a_blk_elems[6] = _mm256_blend_ps(mat_a_blk_elems[2], mat_a_blk_elems[7], 0xCC); - mat_a_blk_elems[7] = _mm256_blend_ps(mat_a_blk_elems[3], mat_a_blk_elems[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_a_blk_elems[4], mat_a_blk_elems[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_a_blk_elems[5], mat_a_blk_elems[7], 0x31); - /* transpose steps end */ - - //Broadcast A8,0 to A15,0 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i4 = k >> 3; - ptr_l_dup++; - -#if GEMM_ACCUM_A - //(Row8): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[0], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_mul_ps(mat_a_blk_elems[0], mat_b_col[0]); - mat_b_rearr[1] = _mm256_mul_ps(mat_a_blk_elems[1], mat_b_col[0]); - mat_b_rearr[2] = _mm256_mul_ps(mat_a_blk_elems[2], mat_b_col[0]); - mat_b_rearr[3] = _mm256_mul_ps(mat_a_blk_elems[3], mat_b_col[0]); - mat_b_rearr[4] = _mm256_mul_ps(mat_a_blk_elems[4], mat_b_col[0]); - mat_b_rearr[5] = _mm256_mul_ps(mat_a_blk_elems[5], mat_b_col[0]); - mat_b_rearr[6] = _mm256_mul_ps(mat_a_blk_elems[6], mat_b_col[0]); - mat_b_rearr[7] = _mm256_mul_ps(mat_a_blk_elems[7], mat_b_col[0]); -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row9): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[1], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[1], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[1], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,2 to A15,2 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row10): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[2], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[2], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[2], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[2], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,3 to A15,3 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row11): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[3], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[3], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[3], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[3], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[3], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,4 to A15,4 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row12): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[4], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[4], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[4], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[4], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[4], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[4], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,5 to A15,5 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row13): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[5], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[5], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[5], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[5], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[5], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[5], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[5], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,6 to A15,6 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row14): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[6], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[6], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[6], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[6], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[6], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[6], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[6], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[6], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A8,7 to A15,7 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[7] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - ptr_l_dup++; -#if GEMM_ACCUM_A - //(Row15): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[0] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[0] = _mm256_fmadd_ps(mat_a_blk_elems[0], mat_b_col[7], mat_b_rearr[0]);//d = c - (a*b) - mat_b_rearr[1] = _mm256_fmadd_ps(mat_a_blk_elems[1], mat_b_col[7], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fmadd_ps(mat_a_blk_elems[2], mat_b_col[7], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fmadd_ps(mat_a_blk_elems[3], mat_b_col[7], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fmadd_ps(mat_a_blk_elems[4], mat_b_col[7], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fmadd_ps(mat_a_blk_elems[5], mat_b_col[7], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fmadd_ps(mat_a_blk_elems[6], mat_b_col[7], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fmadd_ps(mat_a_blk_elems[7], mat_b_col[7], mat_b_rearr[7]);//d = c - (a*b) -#endif - //end loop of cols - //} - //i2 += cs_b_offset[6]; - i4 += 8; - } - //trsm solve - - k = 0; - //for (i2 = 0; i2 < numCols_b; i2 += 8) - //{ - //i2 = i1 + r; - /////////////////// Complete Lower 8x8 block trsm of B :- lower 8x8 block of B with lower right 8x8 block of A -#if !GEMM_ACCUM_A - //Read 8 cols of B columns of Block-to-be-solved - mat_b_rearr[0] = _mm256_loadu_ps((float const *)ptr_b + i); - mat_b_rearr[1] = _mm256_loadu_ps((float const *)(ptr_b + cs_b + i)); - mat_b_rearr[2] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[0] + i)); - mat_b_rearr[3] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[1] + i)); - mat_b_rearr[4] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[2] + i)); - mat_b_rearr[5] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[3] + i)); - mat_b_rearr[6] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[4] + i)); - mat_b_rearr[7] = _mm256_loadu_ps((float const *)(ptr_b + cs_b_offset[5] + i)); - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ - - mat_b_col[0] = _mm256_mul_ps(mat_b_col[0], alphaReg); - mat_b_col[1] = _mm256_mul_ps(mat_b_col[1], alphaReg); - mat_b_col[2] = _mm256_mul_ps(mat_b_col[2], alphaReg); - mat_b_col[3] = _mm256_mul_ps(mat_b_col[3], alphaReg); - mat_b_col[4] = _mm256_mul_ps(mat_b_col[4], alphaReg); - mat_b_col[5] = _mm256_mul_ps(mat_b_col[5], alphaReg); - mat_b_col[6] = _mm256_mul_ps(mat_b_col[6], alphaReg); - mat_b_col[7] = _mm256_mul_ps(mat_b_col[7], alphaReg); -#endif - //Broadcast A10 to A70 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l)); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[0])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[1])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[2])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[3])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[4])); - mat_a_blk_elems[6] = _mm256_broadcast_ss((float const *)(ptr_l_dup + cs_l_offset[5])); - //i += cs_l; - -#if GEMM_ACCUM_A - //(Row0): already done - -#else - mat_b_rearr[0] = _mm256_sub_ps(mat_b_col[0], mat_b_rearr[0]); -#endif - -#if GEMM_ACCUM_A - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#else - mat_b_rearr[1] = _mm256_sub_ps(mat_b_col[1], mat_b_rearr[1]); - mat_b_rearr[2] = _mm256_sub_ps(mat_b_col[2], mat_b_rearr[2]); - mat_b_rearr[3] = _mm256_sub_ps(mat_b_col[3], mat_b_rearr[3]); - mat_b_rearr[4] = _mm256_sub_ps(mat_b_col[4], mat_b_rearr[4]); - mat_b_rearr[5] = _mm256_sub_ps(mat_b_col[5], mat_b_rearr[5]); - mat_b_rearr[6] = _mm256_sub_ps(mat_b_col[6], mat_b_rearr[6]); - mat_b_rearr[7] = _mm256_sub_ps(mat_b_col[7], mat_b_rearr[7]); - - //(Row1): FMA operations of b1 with elements of indices from (1, 0) uptill (7, 0) - mat_b_rearr[1] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[0], mat_b_rearr[1]);//d = c - (a*b) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[0], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[0], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[0], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[0], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[0], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[6], mat_b_rearr[0], mat_b_rearr[7]);//d = c - (a*b) -#endif - //Broadcast A21 to A71 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[0])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[1])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[2])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[3])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[4])); - mat_a_blk_elems[5] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 1 + cs_l_offset[5])); - //i += cs_l; - - - - //(Row2): FMA operations of b2 with elements of indices from (2, 0) uptill (7, 0) - mat_b_rearr[2] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[1], mat_b_rearr[2]);//d = c - (a*b) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[1], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[1], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[1], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[1], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[5], mat_b_rearr[1], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A32 to A72 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[1])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[2])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[3])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[4])); - mat_a_blk_elems[4] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 2 + cs_l_offset[5])); - //i += cs_l; - - - - //(Row3): FMA operations of b3 with elements of indices from (3, 0) uptill (7, 0) - mat_b_rearr[3] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[2], mat_b_rearr[3]);//d = c - (a*b) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[2], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[2], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[2], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[4], mat_b_rearr[2], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A43 to A73 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[2])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[3])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[4])); - mat_a_blk_elems[3] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 3 + cs_l_offset[5])); - //i += cs_l; - - - - //(Row4): FMA operations of b4 with elements of indices from (4, 0) uptill (7, 0) - mat_b_rearr[4] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[3], mat_b_rearr[4]);//d = c - (a*b) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[3], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[3], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[3], mat_b_rearr[3], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A54 to A74 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[3])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[4])); - mat_a_blk_elems[2] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 4 + cs_l_offset[5])); - //i += cs_l; - - - - //(Row5): FMA operations of b5 with elements of indices from (5, 0) uptill (7, 0) - mat_b_rearr[5] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[4], mat_b_rearr[5]);//d = c - (a*b) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[4], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[2], mat_b_rearr[4], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A65 to A75 to registers - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[4])); - mat_a_blk_elems[1] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 5 + cs_l_offset[5])); - //i += cs_l; - - - - //(Row6): FMA operations of b6 with elements of indices from (6, 0) uptill (7, 0) - mat_b_rearr[6] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[5], mat_b_rearr[6]);//d = c - (a*b) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[1], mat_b_rearr[5], mat_b_rearr[7]);//d = c - (a*b) - - //Broadcast A76 to register - mat_a_blk_elems[0] = _mm256_broadcast_ss((float const *)(ptr_l_dup + 6 + cs_l_offset[5])); - - - - //(Row7): FMA operations of b7 with elements of index (7, 0) - mat_b_rearr[7] = _mm256_fnmadd_ps(mat_a_blk_elems[0], mat_b_rearr[6], mat_b_rearr[7]);//d = c - (a*b) - - - - //////////////////////////////////////////////////////////////////////////////// - - /* transpose steps start */ - ////unpacklow//// - mat_b_col[0] = _mm256_unpacklo_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_col[1] = _mm256_unpacklo_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_col[2] = _mm256_unpacklo_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_col[3] = _mm256_unpacklo_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange low elements -#if REARRANGE_SHFL == 1 - mat_b_col[4] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x44); - mat_b_col[5] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0xEE); - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x44); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0xEE); -#else - mat_b_col[6] = _mm256_shuffle_ps(mat_b_col[0], mat_b_col[1], 0x4E); - mat_b_col[7] = _mm256_shuffle_ps(mat_b_col[2], mat_b_col[3], 0x4E); - mat_b_col[4] = _mm256_blend_ps(mat_b_col[0], mat_b_col[6], 0xCC); - mat_b_col[5] = _mm256_blend_ps(mat_b_col[1], mat_b_col[6], 0x33); - mat_b_col[6] = _mm256_blend_ps(mat_b_col[2], mat_b_col[7], 0xCC); - mat_b_col[7] = _mm256_blend_ps(mat_b_col[3], mat_b_col[7], 0x33); -#endif - //Merge rearranged low elements into complete rows - mat_b_col[0] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x20); - mat_b_col[4] = _mm256_permute2f128_ps(mat_b_col[4], mat_b_col[6], 0x31); - mat_b_col[1] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x20); - mat_b_col[5] = _mm256_permute2f128_ps(mat_b_col[5], mat_b_col[7], 0x31); - - ////unpackhigh//// - mat_b_rearr[0] = _mm256_unpackhi_ps(mat_b_rearr[0], mat_b_rearr[1]); - mat_b_rearr[1] = _mm256_unpackhi_ps(mat_b_rearr[2], mat_b_rearr[3]); - mat_b_rearr[2] = _mm256_unpackhi_ps(mat_b_rearr[4], mat_b_rearr[5]); - mat_b_rearr[3] = _mm256_unpackhi_ps(mat_b_rearr[6], mat_b_rearr[7]); - - //Rearrange high elements -#if REARRANGE_SHFL == 1 - mat_b_rearr[4] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x44); - mat_b_rearr[5] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0xEE); - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x44); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0xEE); -#else - mat_b_rearr[6] = _mm256_shuffle_ps(mat_b_rearr[0], mat_b_rearr[1], 0x4E); - mat_b_rearr[7] = _mm256_shuffle_ps(mat_b_rearr[2], mat_b_rearr[3], 0x4E); - mat_b_rearr[4] = _mm256_blend_ps(mat_b_rearr[0], mat_b_rearr[6], 0xCC); - mat_b_rearr[5] = _mm256_blend_ps(mat_b_rearr[1], mat_b_rearr[6], 0x33); - mat_b_rearr[6] = _mm256_blend_ps(mat_b_rearr[2], mat_b_rearr[7], 0xCC); - mat_b_rearr[7] = _mm256_blend_ps(mat_b_rearr[3], mat_b_rearr[7], 0x33); -#endif - - //Merge rearranged high elements into complete rows - mat_b_col[2] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x20); - mat_b_col[6] = _mm256_permute2f128_ps(mat_b_rearr[4], mat_b_rearr[6], 0x31); - mat_b_col[3] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x20); - mat_b_col[7] = _mm256_permute2f128_ps(mat_b_rearr[5], mat_b_rearr[7], 0x31); - /* transpose steps end */ - - //Store the computed B columns - _mm256_storeu_ps((float *)ptr_b_dup + i2, mat_b_col[0]); - _mm256_storeu_ps((float *)(ptr_b_dup + (cs_b)+i2), mat_b_col[1]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[0] + i2), mat_b_col[2]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[1] + i2), mat_b_col[3]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[2] + i2), mat_b_col[4]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[3] + i2), mat_b_col[5]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[4] + i2), mat_b_col[6]); - _mm256_storeu_ps((float *)(ptr_b_dup + cs_b_offset[5] + i2), mat_b_col[7]); - //printf("writing B => m[%d], n[%d], [%f]\n", j, k, *(ptr_b_dup + k)); - k++; - //} - i += cs_b_offset[6]; - i2 += cs_b_offset[6]; - } - } //numRows of A - ///////////////////loop ends ///////////////////// -} -#endif From 13806ba3b01ca0dd341f4720fb930f97e46710b0 Mon Sep 17 00:00:00 2001 From: kdevraje Date: Mon, 27 May 2019 16:24:43 +0530 Subject: [PATCH 45/53] This check in has changes w.r.t Copyright information, which is changed to (start year) - 2019 Change-Id: Ide3c8f7172210b8d3538d3c36e88634ab1ba9041 --- LICENSE | 2 +- build/bli_config.h.in | 2 +- build/detect/config/config_detect.c | 2 +- build/detect/config/old/cpuid_x86.c | 2 +- build/irun.py | 2 +- config/zen/bli_cntx_init_zen.c | 2 +- config/zen/bli_family_zen.h | 2 +- config/zen/old/bli_kernel.h | 2 +- configure | 2 +- frame/1m/packm/bli_packm.h | 2 +- frame/1m/packm/bli_packm_blk_var1.c | 2 +- frame/1m/packm/bli_packm_cntl.c | 2 +- frame/1m/packm/bli_packm_cntl.h | 2 +- frame/1m/packm/bli_packm_thrinfo.c | 2 +- frame/1m/packm/bli_packm_thrinfo.h | 2 +- frame/1m/packm/bli_packm_var.h | 2 +- frame/1m/unpackm/bli_unpackm_cntl.c | 2 +- frame/1m/unpackm/bli_unpackm_cntl.h | 2 +- frame/3/bli_l3_cntl.c | 2 +- frame/3/bli_l3_cntl.h | 2 +- frame/3/bli_l3_packm.c | 2 +- frame/3/bli_l3_packm.h | 2 +- frame/3/bli_l3_thrinfo.c | 2 +- frame/3/bli_l3_thrinfo.h | 2 +- frame/3/gemm/bli_gemm_blk_var1.c | 2 +- frame/3/gemm/bli_gemm_blk_var2.c | 2 +- frame/3/gemm/bli_gemm_cntl.c | 2 +- frame/3/gemm/bli_gemm_cntl.h | 2 +- frame/3/gemm/bli_gemm_front.c | 2 +- frame/3/gemm/bli_gemm_int.c | 2 +- frame/3/gemm/bli_gemm_ker_var1.c | 2 +- frame/3/gemm/bli_gemm_ker_var2.c | 2 +- frame/3/gemm/bli_gemm_md.c | 2 +- frame/3/gemm/bli_gemm_var.h | 2 +- frame/3/gemm/ind/bli_gemm4mb_ker_var2.c | 2 +- frame/3/gemm/other/bli_gemm_ker_var2.c | 2 +- frame/3/gemm/other/bli_gemm_ker_var2rr.c | 2 +- frame/3/gemm/other/bli_gemm_ker_var2sl.c | 2 +- frame/3/herk/bli_herk_l_ker_var2.c | 2 +- frame/3/herk/bli_herk_u_ker_var2.c | 2 +- frame/3/herk/bli_herk_var.h | 2 +- frame/3/herk/bli_herk_x_ker_var2.c | 2 +- frame/3/herk/other/bli_herk_l_ker_var2.1looprr.c | 2 +- frame/3/herk/other/bli_herk_l_ker_var2.c | 2 +- frame/3/herk/other/bli_herk_l_ker_var2rr.c | 2 +- frame/3/herk/other/bli_herk_l_ker_var2sl.c | 2 +- frame/3/herk/other/bli_herk_u_ker_var2.1looprr.c | 2 +- frame/3/herk/other/bli_herk_u_ker_var2.c | 2 +- frame/3/herk/other/bli_herk_u_ker_var2rr.c | 2 +- frame/3/herk/other/bli_herk_u_ker_var2sl.c | 2 +- frame/3/syrk/bli_syrk_front.c | 2 +- frame/3/trmm/bli_trmm_front.c | 2 +- frame/3/trmm/bli_trmm_ll_ker_var2.c | 2 +- frame/3/trmm/bli_trmm_lu_ker_var2.c | 2 +- frame/3/trmm/bli_trmm_rl_ker_var2.c | 2 +- frame/3/trmm/bli_trmm_ru_ker_var2.c | 2 +- frame/3/trmm/bli_trmm_var.h | 2 +- frame/3/trmm/bli_trmm_xx_ker_var2.c | 2 +- frame/3/trmm/other/bli_trmm_ll_ker_var2.c | 2 +- frame/3/trmm/other/bli_trmm_ll_ker_var2rr.c | 2 +- frame/3/trmm/other/bli_trmm_ll_ker_var2sl.c | 2 +- frame/3/trmm/other/bli_trmm_lu_ker_var2.c | 2 +- frame/3/trmm/other/bli_trmm_lu_ker_var2rr.c | 2 +- frame/3/trmm/other/bli_trmm_lu_ker_var2sl.c | 2 +- frame/3/trmm/other/bli_trmm_rl_ker_var2.c | 2 +- frame/3/trmm/other/bli_trmm_rl_ker_var2rr.c | 2 +- frame/3/trmm/other/bli_trmm_rl_ker_var2sl.c | 2 +- frame/3/trmm/other/bli_trmm_ru_ker_var2.c | 2 +- frame/3/trmm/other/bli_trmm_ru_ker_var2rr.c | 2 +- frame/3/trmm/other/bli_trmm_ru_ker_var2sl.c | 2 +- frame/3/trsm/bli_trsm_blk_var1.c | 2 +- frame/3/trsm/bli_trsm_blk_var2.c | 2 +- frame/3/trsm/bli_trsm_cntl.c | 2 +- frame/3/trsm/bli_trsm_cntl.h | 2 +- frame/3/trsm/bli_trsm_front.c | 2 +- frame/3/trsm/bli_trsm_front.h | 2 +- frame/3/trsm/bli_trsm_ll_ker_var2.c | 2 +- frame/3/trsm/bli_trsm_lu_ker_var2.c | 2 +- frame/3/trsm/bli_trsm_rl_ker_var2.c | 2 +- frame/3/trsm/bli_trsm_ru_ker_var2.c | 2 +- frame/3/trsm/bli_trsm_var.h | 2 +- frame/3/trsm/bli_trsm_xx_ker_var2.c | 2 +- frame/3/trsm/other/bli_trsm_ll_ker_var2.c | 2 +- frame/3/trsm/other/bli_trsm_ll_ker_var2rr.c | 2 +- frame/3/trsm/other/bli_trsm_ll_ker_var2sl.c | 2 +- frame/3/trsm/other/bli_trsm_lu_ker_var2.c | 2 +- frame/3/trsm/other/bli_trsm_lu_ker_var2rr.c | 2 +- frame/3/trsm/other/bli_trsm_lu_ker_var2sl.c | 2 +- frame/3/trsm/other/bli_trsm_rl_ker_var2.c | 2 +- frame/3/trsm/other/bli_trsm_ru_ker_var2.c | 2 +- frame/base/bli_apool.c | 2 +- frame/base/bli_apool.h | 2 +- frame/base/bli_array.c | 2 +- frame/base/bli_array.h | 2 +- frame/base/bli_check.c | 2 +- frame/base/bli_check.h | 2 +- frame/base/bli_clock.c | 2 +- frame/base/bli_cntl.c | 2 +- frame/base/bli_cntl.h | 2 +- frame/base/bli_cntx.c | 2 +- frame/base/bli_cpuid.c | 2 +- frame/base/bli_error.c | 2 +- frame/base/bli_getopt.c | 2 +- frame/base/bli_info.c | 2 +- frame/base/bli_info.h | 2 +- frame/base/bli_init.c | 2 +- frame/base/bli_malloc.c | 2 +- frame/base/bli_malloc.h | 2 +- frame/base/bli_mem.h | 2 +- frame/base/bli_membrk.c | 2 +- frame/base/bli_membrk.h | 2 +- frame/base/bli_memsys.c | 2 +- frame/base/bli_memsys.h | 2 +- frame/base/bli_obj.c | 2 +- frame/base/bli_pool.c | 2 +- frame/base/bli_pool.h | 2 +- frame/base/bli_prune.c | 2 +- frame/base/bli_rntm.h | 2 +- frame/base/bli_sba.c | 2 +- frame/base/bli_sba.h | 2 +- frame/include/bli_macro_defs.h | 2 +- frame/include/bli_param_macro_defs.h | 2 +- frame/include/bli_system.h | 2 +- frame/include/blis.h | 2 +- frame/ind/bli_l3_ind.c | 2 +- frame/ind/oapi/bli_l3_3m4m1m_oapi.c | 2 +- frame/ind/oapi/bli_l3_ind_oapi.c | 2 +- frame/ind/oapi/bli_l3_nat_oapi.c | 2 +- frame/thread/bli_pthread.c | 2 +- frame/thread/bli_thrcomm.c | 2 +- frame/thread/bli_thrcomm.h | 2 +- frame/thread/bli_thrcomm_openmp.c | 2 +- frame/thread/bli_thrcomm_openmp.h | 2 +- frame/thread/bli_thrcomm_pthreads.c | 2 +- frame/thread/bli_thrcomm_single.c | 2 +- frame/thread/bli_thread.c | 2 +- frame/thread/bli_thread.h | 2 +- frame/thread/bli_thrinfo.c | 2 +- frame/thread/bli_thrinfo.h | 2 +- frame/thread/old/bli_mutex.h | 2 +- frame/thread/old/bli_mutex_openmp.h | 2 +- frame/thread/old/bli_mutex_pthreads.h | 2 +- frame/thread/old/bli_mutex_single.h | 2 +- frame/util/bli_util_unb_var1.c | 2 +- kernels/haswell/3/bli_gemm_haswell_asm_d6x8.c | 2 +- kernels/haswell/3/bli_gemmtrsm_l_haswell_asm_d6x8.c | 2 +- kernels/haswell/3/bli_gemmtrsm_u_haswell_asm_d6x8.c | 2 +- kernels/zen/1/bli_amaxv_zen_int.c | 2 +- kernels/zen/1/bli_axpyv_zen_int.c | 2 +- kernels/zen/1/bli_axpyv_zen_int10.c | 2 +- kernels/zen/1/bli_dotv_zen_int.c | 2 +- kernels/zen/1/bli_dotv_zen_int10.c | 2 +- kernels/zen/1/bli_dotxv_zen_int.c | 2 +- kernels/zen/1/bli_scalv_zen_int.c | 2 +- kernels/zen/1/bli_scalv_zen_int10.c | 2 +- kernels/zen/1f/bli_axpyf_zen_int_8.c | 2 +- kernels/zen/1f/bli_dotxf_zen_int_8.c | 2 +- kernels/zen/3/bli_gemm_small.c | 2 +- kernels/zen/3/bli_syrk_small.c | 2 +- ref_kernels/bli_cntx_ref.c | 2 +- sandbox/ref99/blx_gemm_front.c | 2 +- sandbox/ref99/blx_gemm_int.c | 2 +- sandbox/ref99/cntl/blx_gemm_cntl.c | 2 +- sandbox/ref99/oapi/bli_gemmnat.c | 2 +- sandbox/ref99/vars/blx_gemm_blk_var1.c | 2 +- sandbox/ref99/vars/blx_gemm_blk_var2.c | 2 +- sandbox/ref99/vars/blx_gemm_ker_var2.c | 2 +- sandbox/ref99/vars/blx_gemm_var.h | 2 +- sandbox/ref99/vars/other/blx_gemm_ker_var2rr.c | 2 +- sandbox/ref99/vars/other/blx_gemm_ker_var2sl.c | 2 +- test/3m4m/Makefile | 2 +- test/3m4m/test_herk.c | 2 +- test/3m4m/test_trmm.c | 2 +- test/3m4m/test_trsm.c | 2 +- test/Makefile | 2 +- test/test_axpyv.c | 2 +- test/test_dotv.c | 2 +- test/thread_ranges/test_ranges.c | 2 +- testsuite/src/test_addm.c | 2 +- testsuite/src/test_addm.h | 2 +- testsuite/src/test_addv.c | 2 +- testsuite/src/test_addv.h | 2 +- testsuite/src/test_amaxv.c | 2 +- testsuite/src/test_amaxv.h | 2 +- testsuite/src/test_axpbyv.c | 2 +- testsuite/src/test_axpbyv.h | 2 +- testsuite/src/test_axpy2v.c | 2 +- testsuite/src/test_axpy2v.h | 2 +- testsuite/src/test_axpyf.c | 2 +- testsuite/src/test_axpyf.h | 2 +- testsuite/src/test_axpym.c | 2 +- testsuite/src/test_axpym.h | 2 +- testsuite/src/test_axpyv.c | 2 +- testsuite/src/test_axpyv.h | 2 +- testsuite/src/test_copym.c | 2 +- testsuite/src/test_copym.h | 2 +- testsuite/src/test_copyv.c | 2 +- testsuite/src/test_copyv.h | 2 +- testsuite/src/test_dotaxpyv.c | 2 +- testsuite/src/test_dotaxpyv.h | 2 +- testsuite/src/test_dotv.c | 2 +- testsuite/src/test_dotv.h | 2 +- testsuite/src/test_dotxaxpyf.c | 2 +- testsuite/src/test_dotxaxpyf.h | 2 +- testsuite/src/test_dotxf.c | 2 +- testsuite/src/test_dotxf.h | 2 +- testsuite/src/test_dotxv.c | 2 +- testsuite/src/test_dotxv.h | 2 +- testsuite/src/test_gemm.c | 2 +- testsuite/src/test_gemm.h | 2 +- testsuite/src/test_gemm_ukr.c | 2 +- testsuite/src/test_gemm_ukr.h | 2 +- testsuite/src/test_gemmtrsm_ukr.c | 2 +- testsuite/src/test_gemmtrsm_ukr.h | 2 +- testsuite/src/test_gemv.c | 2 +- testsuite/src/test_gemv.h | 2 +- testsuite/src/test_ger.c | 2 +- testsuite/src/test_ger.h | 2 +- testsuite/src/test_hemm.c | 2 +- testsuite/src/test_hemm.h | 2 +- testsuite/src/test_hemv.c | 2 +- testsuite/src/test_hemv.h | 2 +- testsuite/src/test_her.c | 2 +- testsuite/src/test_her.h | 2 +- testsuite/src/test_her2.c | 2 +- testsuite/src/test_her2.h | 2 +- testsuite/src/test_her2k.c | 2 +- testsuite/src/test_her2k.h | 2 +- testsuite/src/test_herk.c | 2 +- testsuite/src/test_herk.h | 2 +- testsuite/src/test_libblis.c | 2 +- testsuite/src/test_libblis.h | 2 +- testsuite/src/test_normfm.c | 2 +- testsuite/src/test_normfm.h | 2 +- testsuite/src/test_normfv.c | 2 +- testsuite/src/test_normfv.h | 2 +- testsuite/src/test_randm.c | 2 +- testsuite/src/test_randm.h | 2 +- testsuite/src/test_randv.c | 2 +- testsuite/src/test_randv.h | 2 +- testsuite/src/test_scal2m.c | 2 +- testsuite/src/test_scal2m.h | 2 +- testsuite/src/test_scal2v.c | 2 +- testsuite/src/test_scal2v.h | 2 +- testsuite/src/test_scalm.c | 2 +- testsuite/src/test_scalm.h | 2 +- testsuite/src/test_scalv.c | 2 +- testsuite/src/test_scalv.h | 2 +- testsuite/src/test_setm.c | 2 +- testsuite/src/test_setm.h | 2 +- testsuite/src/test_setv.c | 2 +- testsuite/src/test_setv.h | 2 +- testsuite/src/test_subm.c | 2 +- testsuite/src/test_subm.h | 2 +- testsuite/src/test_subv.c | 2 +- testsuite/src/test_subv.h | 2 +- testsuite/src/test_symm.c | 2 +- testsuite/src/test_symm.h | 2 +- testsuite/src/test_symv.c | 2 +- testsuite/src/test_symv.h | 2 +- testsuite/src/test_syr.c | 2 +- testsuite/src/test_syr.h | 2 +- testsuite/src/test_syr2.c | 2 +- testsuite/src/test_syr2.h | 2 +- testsuite/src/test_syr2k.c | 2 +- testsuite/src/test_syr2k.h | 2 +- testsuite/src/test_syrk.c | 2 +- testsuite/src/test_syrk.h | 2 +- testsuite/src/test_trmm.c | 2 +- testsuite/src/test_trmm.h | 2 +- testsuite/src/test_trmm3.c | 2 +- testsuite/src/test_trmm3.h | 2 +- testsuite/src/test_trmv.c | 2 +- testsuite/src/test_trmv.h | 2 +- testsuite/src/test_trsm.c | 2 +- testsuite/src/test_trsm.h | 2 +- testsuite/src/test_trsm_ukr.c | 2 +- testsuite/src/test_trsm_ukr.h | 2 +- testsuite/src/test_trsv.c | 2 +- testsuite/src/test_trsv.h | 2 +- testsuite/src/test_xpbyv.c | 2 +- testsuite/src/test_xpbyv.h | 2 +- 282 files changed, 282 insertions(+), 282 deletions(-) diff --git a/LICENSE b/LICENSE index 1c6845375..b9cde54b8 100644 --- a/LICENSE +++ b/LICENSE @@ -15,7 +15,7 @@ copyright info. All parties provide their portions of the code under the Copyright (C) 2018, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP -Copyright (C) 2018, Advanced Micro Devices, Inc. +Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/build/bli_config.h.in b/build/bli_config.h.in index e3a97ca79..f940090ad 100644 --- a/build/bli_config.h.in +++ b/build/bli_config.h.in @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/build/detect/config/config_detect.c b/build/detect/config/config_detect.c index 85b79e476..7d0e95522 100644 --- a/build/detect/config/config_detect.c +++ b/build/detect/config/config_detect.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/build/detect/config/old/cpuid_x86.c b/build/detect/config/old/cpuid_x86.c index ab71708f3..f4985e391 100644 --- a/build/detect/config/old/cpuid_x86.c +++ b/build/detect/config/old/cpuid_x86.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2015, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/build/irun.py b/build/irun.py index d9d1e6b77..429981603 100755 --- a/build/irun.py +++ b/build/irun.py @@ -5,7 +5,7 @@ # libraries. # # Copyright (C) 2018, The University of Texas at Austin -# Copyright (C) 2018, Advanced Micro Devices, Inc. +# Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are diff --git a/config/zen/bli_cntx_init_zen.c b/config/zen/bli_cntx_init_zen.c index 049be6837..7f195de24 100644 --- a/config/zen/bli_cntx_init_zen.c +++ b/config/zen/bli_cntx_init_zen.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/config/zen/bli_family_zen.h b/config/zen/bli_family_zen.h index 67a0806c9..7721469c2 100644 --- a/config/zen/bli_family_zen.h +++ b/config/zen/bli_family_zen.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/config/zen/old/bli_kernel.h b/config/zen/old/bli_kernel.h index 68b9e88e0..cd324fd9a 100644 --- a/config/zen/old/bli_kernel.h +++ b/config/zen/old/bli_kernel.h @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/configure b/configure index 755cf61fc..e72831d8f 100755 --- a/configure +++ b/configure @@ -5,7 +5,7 @@ # libraries. # # Copyright (C) 2014, The University of Texas at Austin -# Copyright (C) 2018, Advanced Micro Devices, Inc. +# Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are diff --git a/frame/1m/packm/bli_packm.h b/frame/1m/packm/bli_packm.h index fbc02e392..85f701165 100644 --- a/frame/1m/packm/bli_packm.h +++ b/frame/1m/packm/bli_packm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/packm/bli_packm_blk_var1.c b/frame/1m/packm/bli_packm_blk_var1.c index 3f753a914..6afc48fad 100644 --- a/frame/1m/packm/bli_packm_blk_var1.c +++ b/frame/1m/packm/bli_packm_blk_var1.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/packm/bli_packm_cntl.c b/frame/1m/packm/bli_packm_cntl.c index 12083f3be..5321d873a 100644 --- a/frame/1m/packm/bli_packm_cntl.c +++ b/frame/1m/packm/bli_packm_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/packm/bli_packm_cntl.h b/frame/1m/packm/bli_packm_cntl.h index fef603ab0..bbf0ea332 100644 --- a/frame/1m/packm/bli_packm_cntl.h +++ b/frame/1m/packm/bli_packm_cntl.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/packm/bli_packm_thrinfo.c b/frame/1m/packm/bli_packm_thrinfo.c index 800fa3c7f..203356b64 100644 --- a/frame/1m/packm/bli_packm_thrinfo.c +++ b/frame/1m/packm/bli_packm_thrinfo.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/packm/bli_packm_thrinfo.h b/frame/1m/packm/bli_packm_thrinfo.h index 190d3f2d3..7a7b04013 100644 --- a/frame/1m/packm/bli_packm_thrinfo.h +++ b/frame/1m/packm/bli_packm_thrinfo.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/packm/bli_packm_var.h b/frame/1m/packm/bli_packm_var.h index 6c11b19ab..ce2065fd1 100644 --- a/frame/1m/packm/bli_packm_var.h +++ b/frame/1m/packm/bli_packm_var.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/unpackm/bli_unpackm_cntl.c b/frame/1m/unpackm/bli_unpackm_cntl.c index 46392269f..7c634a11e 100644 --- a/frame/1m/unpackm/bli_unpackm_cntl.c +++ b/frame/1m/unpackm/bli_unpackm_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/1m/unpackm/bli_unpackm_cntl.h b/frame/1m/unpackm/bli_unpackm_cntl.h index b282c3561..49a8b19f6 100644 --- a/frame/1m/unpackm/bli_unpackm_cntl.h +++ b/frame/1m/unpackm/bli_unpackm_cntl.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/bli_l3_cntl.c b/frame/3/bli_l3_cntl.c index efdca53db..f6bfbedbb 100644 --- a/frame/3/bli_l3_cntl.c +++ b/frame/3/bli_l3_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/bli_l3_cntl.h b/frame/3/bli_l3_cntl.h index 0c04f348c..c308c8a96 100644 --- a/frame/3/bli_l3_cntl.h +++ b/frame/3/bli_l3_cntl.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/bli_l3_packm.c b/frame/3/bli_l3_packm.c index bfb066bfb..b5dc17d4a 100644 --- a/frame/3/bli_l3_packm.c +++ b/frame/3/bli_l3_packm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/bli_l3_packm.h b/frame/3/bli_l3_packm.h index 37b1db105..696dabf59 100644 --- a/frame/3/bli_l3_packm.h +++ b/frame/3/bli_l3_packm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/bli_l3_thrinfo.c b/frame/3/bli_l3_thrinfo.c index 5a6228609..317a1cbf4 100644 --- a/frame/3/bli_l3_thrinfo.c +++ b/frame/3/bli_l3_thrinfo.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/bli_l3_thrinfo.h b/frame/3/bli_l3_thrinfo.h index 72a4f77e2..a01392d18 100644 --- a/frame/3/bli_l3_thrinfo.h +++ b/frame/3/bli_l3_thrinfo.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_blk_var1.c b/frame/3/gemm/bli_gemm_blk_var1.c index 07c029c6a..ae82578f1 100644 --- a/frame/3/gemm/bli_gemm_blk_var1.c +++ b/frame/3/gemm/bli_gemm_blk_var1.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_blk_var2.c b/frame/3/gemm/bli_gemm_blk_var2.c index 33431a836..9200d7f0e 100644 --- a/frame/3/gemm/bli_gemm_blk_var2.c +++ b/frame/3/gemm/bli_gemm_blk_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_cntl.c b/frame/3/gemm/bli_gemm_cntl.c index 67c71e798..fc0a4a786 100644 --- a/frame/3/gemm/bli_gemm_cntl.c +++ b/frame/3/gemm/bli_gemm_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_cntl.h b/frame/3/gemm/bli_gemm_cntl.h index e19384a51..c6d20b170 100644 --- a/frame/3/gemm/bli_gemm_cntl.h +++ b/frame/3/gemm/bli_gemm_cntl.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_front.c b/frame/3/gemm/bli_gemm_front.c index 0d83104e1..026016f26 100644 --- a/frame/3/gemm/bli_gemm_front.c +++ b/frame/3/gemm/bli_gemm_front.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_int.c b/frame/3/gemm/bli_gemm_int.c index 25a6215df..5128bea69 100644 --- a/frame/3/gemm/bli_gemm_int.c +++ b/frame/3/gemm/bli_gemm_int.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_ker_var1.c b/frame/3/gemm/bli_gemm_ker_var1.c index 4dcffd279..096091e76 100644 --- a/frame/3/gemm/bli_gemm_ker_var1.c +++ b/frame/3/gemm/bli_gemm_ker_var1.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_ker_var2.c b/frame/3/gemm/bli_gemm_ker_var2.c index 41bb3f455..e9b43f2ac 100644 --- a/frame/3/gemm/bli_gemm_ker_var2.c +++ b/frame/3/gemm/bli_gemm_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_md.c b/frame/3/gemm/bli_gemm_md.c index b08bea2c4..1f5a8e593 100644 --- a/frame/3/gemm/bli_gemm_md.c +++ b/frame/3/gemm/bli_gemm_md.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/bli_gemm_var.h b/frame/3/gemm/bli_gemm_var.h index 34cf95ae6..b08271e9b 100644 --- a/frame/3/gemm/bli_gemm_var.h +++ b/frame/3/gemm/bli_gemm_var.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/ind/bli_gemm4mb_ker_var2.c b/frame/3/gemm/ind/bli_gemm4mb_ker_var2.c index e4b377b37..cbc3a1929 100644 --- a/frame/3/gemm/ind/bli_gemm4mb_ker_var2.c +++ b/frame/3/gemm/ind/bli_gemm4mb_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/other/bli_gemm_ker_var2.c b/frame/3/gemm/other/bli_gemm_ker_var2.c index 6ae8df0c1..73ca61181 100644 --- a/frame/3/gemm/other/bli_gemm_ker_var2.c +++ b/frame/3/gemm/other/bli_gemm_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/other/bli_gemm_ker_var2rr.c b/frame/3/gemm/other/bli_gemm_ker_var2rr.c index a213e50fc..f79f1311e 100644 --- a/frame/3/gemm/other/bli_gemm_ker_var2rr.c +++ b/frame/3/gemm/other/bli_gemm_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/gemm/other/bli_gemm_ker_var2sl.c b/frame/3/gemm/other/bli_gemm_ker_var2sl.c index 0d710bd73..48ef2791f 100644 --- a/frame/3/gemm/other/bli_gemm_ker_var2sl.c +++ b/frame/3/gemm/other/bli_gemm_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/bli_herk_l_ker_var2.c b/frame/3/herk/bli_herk_l_ker_var2.c index d077b8f89..d09424d9a 100644 --- a/frame/3/herk/bli_herk_l_ker_var2.c +++ b/frame/3/herk/bli_herk_l_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/bli_herk_u_ker_var2.c b/frame/3/herk/bli_herk_u_ker_var2.c index b20a96df7..51fd5d4d6 100644 --- a/frame/3/herk/bli_herk_u_ker_var2.c +++ b/frame/3/herk/bli_herk_u_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/bli_herk_var.h b/frame/3/herk/bli_herk_var.h index 3c565e1b0..00b85fc5c 100644 --- a/frame/3/herk/bli_herk_var.h +++ b/frame/3/herk/bli_herk_var.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/bli_herk_x_ker_var2.c b/frame/3/herk/bli_herk_x_ker_var2.c index c0ce23255..fb4048cd2 100644 --- a/frame/3/herk/bli_herk_x_ker_var2.c +++ b/frame/3/herk/bli_herk_x_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_l_ker_var2.1looprr.c b/frame/3/herk/other/bli_herk_l_ker_var2.1looprr.c index 38675b11b..7238716a7 100644 --- a/frame/3/herk/other/bli_herk_l_ker_var2.1looprr.c +++ b/frame/3/herk/other/bli_herk_l_ker_var2.1looprr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_l_ker_var2.c b/frame/3/herk/other/bli_herk_l_ker_var2.c index 904da9f5e..650acf0d7 100644 --- a/frame/3/herk/other/bli_herk_l_ker_var2.c +++ b/frame/3/herk/other/bli_herk_l_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_l_ker_var2rr.c b/frame/3/herk/other/bli_herk_l_ker_var2rr.c index a313f04b2..19edea32b 100644 --- a/frame/3/herk/other/bli_herk_l_ker_var2rr.c +++ b/frame/3/herk/other/bli_herk_l_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_l_ker_var2sl.c b/frame/3/herk/other/bli_herk_l_ker_var2sl.c index f913cced2..68e9de181 100644 --- a/frame/3/herk/other/bli_herk_l_ker_var2sl.c +++ b/frame/3/herk/other/bli_herk_l_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_u_ker_var2.1looprr.c b/frame/3/herk/other/bli_herk_u_ker_var2.1looprr.c index cd4a4e7ad..f1b191a7e 100644 --- a/frame/3/herk/other/bli_herk_u_ker_var2.1looprr.c +++ b/frame/3/herk/other/bli_herk_u_ker_var2.1looprr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_u_ker_var2.c b/frame/3/herk/other/bli_herk_u_ker_var2.c index 0bdc0b0a4..7fbc90fed 100644 --- a/frame/3/herk/other/bli_herk_u_ker_var2.c +++ b/frame/3/herk/other/bli_herk_u_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_u_ker_var2rr.c b/frame/3/herk/other/bli_herk_u_ker_var2rr.c index 4ffa8085c..1643bbc39 100644 --- a/frame/3/herk/other/bli_herk_u_ker_var2rr.c +++ b/frame/3/herk/other/bli_herk_u_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/herk/other/bli_herk_u_ker_var2sl.c b/frame/3/herk/other/bli_herk_u_ker_var2sl.c index 7af7ee56d..24db2a4a3 100644 --- a/frame/3/herk/other/bli_herk_u_ker_var2sl.c +++ b/frame/3/herk/other/bli_herk_u_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/syrk/bli_syrk_front.c b/frame/3/syrk/bli_syrk_front.c index cc2163fae..fbf44d80e 100644 --- a/frame/3/syrk/bli_syrk_front.c +++ b/frame/3/syrk/bli_syrk_front.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/bli_trmm_front.c b/frame/3/trmm/bli_trmm_front.c index aee9d1d6f..fa9a6389d 100644 --- a/frame/3/trmm/bli_trmm_front.c +++ b/frame/3/trmm/bli_trmm_front.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/bli_trmm_ll_ker_var2.c b/frame/3/trmm/bli_trmm_ll_ker_var2.c index 98e62926c..5c7a7bcd0 100644 --- a/frame/3/trmm/bli_trmm_ll_ker_var2.c +++ b/frame/3/trmm/bli_trmm_ll_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/bli_trmm_lu_ker_var2.c b/frame/3/trmm/bli_trmm_lu_ker_var2.c index 624604141..06705da44 100644 --- a/frame/3/trmm/bli_trmm_lu_ker_var2.c +++ b/frame/3/trmm/bli_trmm_lu_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/bli_trmm_rl_ker_var2.c b/frame/3/trmm/bli_trmm_rl_ker_var2.c index 117cf63c5..9a17ed644 100644 --- a/frame/3/trmm/bli_trmm_rl_ker_var2.c +++ b/frame/3/trmm/bli_trmm_rl_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/bli_trmm_ru_ker_var2.c b/frame/3/trmm/bli_trmm_ru_ker_var2.c index ea59959c7..1062b95d8 100644 --- a/frame/3/trmm/bli_trmm_ru_ker_var2.c +++ b/frame/3/trmm/bli_trmm_ru_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/bli_trmm_var.h b/frame/3/trmm/bli_trmm_var.h index 09694ca5c..262b0490f 100644 --- a/frame/3/trmm/bli_trmm_var.h +++ b/frame/3/trmm/bli_trmm_var.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/bli_trmm_xx_ker_var2.c b/frame/3/trmm/bli_trmm_xx_ker_var2.c index 343aaa078..330d06ae8 100644 --- a/frame/3/trmm/bli_trmm_xx_ker_var2.c +++ b/frame/3/trmm/bli_trmm_xx_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_ll_ker_var2.c b/frame/3/trmm/other/bli_trmm_ll_ker_var2.c index 3747a0dcf..1e5f9a9f4 100644 --- a/frame/3/trmm/other/bli_trmm_ll_ker_var2.c +++ b/frame/3/trmm/other/bli_trmm_ll_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_ll_ker_var2rr.c b/frame/3/trmm/other/bli_trmm_ll_ker_var2rr.c index ea979d7c3..3b2575f66 100644 --- a/frame/3/trmm/other/bli_trmm_ll_ker_var2rr.c +++ b/frame/3/trmm/other/bli_trmm_ll_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_ll_ker_var2sl.c b/frame/3/trmm/other/bli_trmm_ll_ker_var2sl.c index e612b340c..1dbd0053a 100644 --- a/frame/3/trmm/other/bli_trmm_ll_ker_var2sl.c +++ b/frame/3/trmm/other/bli_trmm_ll_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_lu_ker_var2.c b/frame/3/trmm/other/bli_trmm_lu_ker_var2.c index 9a4e36b65..17d2f9a2f 100644 --- a/frame/3/trmm/other/bli_trmm_lu_ker_var2.c +++ b/frame/3/trmm/other/bli_trmm_lu_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_lu_ker_var2rr.c b/frame/3/trmm/other/bli_trmm_lu_ker_var2rr.c index 551bc097d..f1a586ce6 100644 --- a/frame/3/trmm/other/bli_trmm_lu_ker_var2rr.c +++ b/frame/3/trmm/other/bli_trmm_lu_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_lu_ker_var2sl.c b/frame/3/trmm/other/bli_trmm_lu_ker_var2sl.c index 132c732d6..26806220a 100644 --- a/frame/3/trmm/other/bli_trmm_lu_ker_var2sl.c +++ b/frame/3/trmm/other/bli_trmm_lu_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_rl_ker_var2.c b/frame/3/trmm/other/bli_trmm_rl_ker_var2.c index b29df0850..d856cc383 100644 --- a/frame/3/trmm/other/bli_trmm_rl_ker_var2.c +++ b/frame/3/trmm/other/bli_trmm_rl_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_rl_ker_var2rr.c b/frame/3/trmm/other/bli_trmm_rl_ker_var2rr.c index 14b235918..80350b401 100644 --- a/frame/3/trmm/other/bli_trmm_rl_ker_var2rr.c +++ b/frame/3/trmm/other/bli_trmm_rl_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_rl_ker_var2sl.c b/frame/3/trmm/other/bli_trmm_rl_ker_var2sl.c index cf4a6e086..a44942da6 100644 --- a/frame/3/trmm/other/bli_trmm_rl_ker_var2sl.c +++ b/frame/3/trmm/other/bli_trmm_rl_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_ru_ker_var2.c b/frame/3/trmm/other/bli_trmm_ru_ker_var2.c index 602f4cc3b..120974ef0 100644 --- a/frame/3/trmm/other/bli_trmm_ru_ker_var2.c +++ b/frame/3/trmm/other/bli_trmm_ru_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_ru_ker_var2rr.c b/frame/3/trmm/other/bli_trmm_ru_ker_var2rr.c index 03eaa6ea6..45a7b5628 100644 --- a/frame/3/trmm/other/bli_trmm_ru_ker_var2rr.c +++ b/frame/3/trmm/other/bli_trmm_ru_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trmm/other/bli_trmm_ru_ker_var2sl.c b/frame/3/trmm/other/bli_trmm_ru_ker_var2sl.c index 2411a24a4..d2b4744e6 100644 --- a/frame/3/trmm/other/bli_trmm_ru_ker_var2sl.c +++ b/frame/3/trmm/other/bli_trmm_ru_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_blk_var1.c b/frame/3/trsm/bli_trsm_blk_var1.c index 01ae6aff9..60b2708ab 100644 --- a/frame/3/trsm/bli_trsm_blk_var1.c +++ b/frame/3/trsm/bli_trsm_blk_var1.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_blk_var2.c b/frame/3/trsm/bli_trsm_blk_var2.c index 1249d9674..f9f6f5f81 100644 --- a/frame/3/trsm/bli_trsm_blk_var2.c +++ b/frame/3/trsm/bli_trsm_blk_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_cntl.c b/frame/3/trsm/bli_trsm_cntl.c index 8ae6f6da2..ad4ea10e5 100644 --- a/frame/3/trsm/bli_trsm_cntl.c +++ b/frame/3/trsm/bli_trsm_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_cntl.h b/frame/3/trsm/bli_trsm_cntl.h index 17b8d3c18..f81f70d07 100644 --- a/frame/3/trsm/bli_trsm_cntl.h +++ b/frame/3/trsm/bli_trsm_cntl.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_front.c b/frame/3/trsm/bli_trsm_front.c index 303570179..3c3b08a08 100644 --- a/frame/3/trsm/bli_trsm_front.c +++ b/frame/3/trsm/bli_trsm_front.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_front.h b/frame/3/trsm/bli_trsm_front.h index cd65e4454..c8942730b 100644 --- a/frame/3/trsm/bli_trsm_front.h +++ b/frame/3/trsm/bli_trsm_front.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_ll_ker_var2.c b/frame/3/trsm/bli_trsm_ll_ker_var2.c index 37823d7bf..0f69057ae 100644 --- a/frame/3/trsm/bli_trsm_ll_ker_var2.c +++ b/frame/3/trsm/bli_trsm_ll_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_lu_ker_var2.c b/frame/3/trsm/bli_trsm_lu_ker_var2.c index 853bccf91..f2cb62608 100644 --- a/frame/3/trsm/bli_trsm_lu_ker_var2.c +++ b/frame/3/trsm/bli_trsm_lu_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_rl_ker_var2.c b/frame/3/trsm/bli_trsm_rl_ker_var2.c index 87e1a0b28..1508cd596 100644 --- a/frame/3/trsm/bli_trsm_rl_ker_var2.c +++ b/frame/3/trsm/bli_trsm_rl_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_ru_ker_var2.c b/frame/3/trsm/bli_trsm_ru_ker_var2.c index 71a72ea24..54288a04d 100644 --- a/frame/3/trsm/bli_trsm_ru_ker_var2.c +++ b/frame/3/trsm/bli_trsm_ru_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_var.h b/frame/3/trsm/bli_trsm_var.h index 0f5f42de8..de7c65936 100644 --- a/frame/3/trsm/bli_trsm_var.h +++ b/frame/3/trsm/bli_trsm_var.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/bli_trsm_xx_ker_var2.c b/frame/3/trsm/bli_trsm_xx_ker_var2.c index dfdcf2eba..2e7577f14 100644 --- a/frame/3/trsm/bli_trsm_xx_ker_var2.c +++ b/frame/3/trsm/bli_trsm_xx_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_ll_ker_var2.c b/frame/3/trsm/other/bli_trsm_ll_ker_var2.c index 1c4b0b5c7..ebd53476d 100644 --- a/frame/3/trsm/other/bli_trsm_ll_ker_var2.c +++ b/frame/3/trsm/other/bli_trsm_ll_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_ll_ker_var2rr.c b/frame/3/trsm/other/bli_trsm_ll_ker_var2rr.c index 3891bffc0..eb7435b4a 100644 --- a/frame/3/trsm/other/bli_trsm_ll_ker_var2rr.c +++ b/frame/3/trsm/other/bli_trsm_ll_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_ll_ker_var2sl.c b/frame/3/trsm/other/bli_trsm_ll_ker_var2sl.c index 1bc2f6e42..df9235740 100644 --- a/frame/3/trsm/other/bli_trsm_ll_ker_var2sl.c +++ b/frame/3/trsm/other/bli_trsm_ll_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_lu_ker_var2.c b/frame/3/trsm/other/bli_trsm_lu_ker_var2.c index 673e1eaa3..97222ffe4 100644 --- a/frame/3/trsm/other/bli_trsm_lu_ker_var2.c +++ b/frame/3/trsm/other/bli_trsm_lu_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_lu_ker_var2rr.c b/frame/3/trsm/other/bli_trsm_lu_ker_var2rr.c index 72761ee54..107aae742 100644 --- a/frame/3/trsm/other/bli_trsm_lu_ker_var2rr.c +++ b/frame/3/trsm/other/bli_trsm_lu_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_lu_ker_var2sl.c b/frame/3/trsm/other/bli_trsm_lu_ker_var2sl.c index 491ae8198..ee18058a4 100644 --- a/frame/3/trsm/other/bli_trsm_lu_ker_var2sl.c +++ b/frame/3/trsm/other/bli_trsm_lu_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_rl_ker_var2.c b/frame/3/trsm/other/bli_trsm_rl_ker_var2.c index 3293289a1..0fff5cc11 100644 --- a/frame/3/trsm/other/bli_trsm_rl_ker_var2.c +++ b/frame/3/trsm/other/bli_trsm_rl_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/3/trsm/other/bli_trsm_ru_ker_var2.c b/frame/3/trsm/other/bli_trsm_ru_ker_var2.c index 9726fd467..f4023ace2 100644 --- a/frame/3/trsm/other/bli_trsm_ru_ker_var2.c +++ b/frame/3/trsm/other/bli_trsm_ru_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_apool.c b/frame/base/bli_apool.c index 5dd98206e..8b8f1f725 100644 --- a/frame/base/bli_apool.c +++ b/frame/base/bli_apool.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_apool.h b/frame/base/bli_apool.h index bc6adf757..7d61e8eb1 100644 --- a/frame/base/bli_apool.h +++ b/frame/base/bli_apool.h @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_array.c b/frame/base/bli_array.c index 3f167056e..6232cffbb 100644 --- a/frame/base/bli_array.c +++ b/frame/base/bli_array.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_array.h b/frame/base/bli_array.h index e3070ae67..cb6db0340 100644 --- a/frame/base/bli_array.h +++ b/frame/base/bli_array.h @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_check.c b/frame/base/bli_check.c index f98f1e685..054247623 100644 --- a/frame/base/bli_check.c +++ b/frame/base/bli_check.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_check.h b/frame/base/bli_check.h index 242dc9487..90de609ad 100644 --- a/frame/base/bli_check.h +++ b/frame/base/bli_check.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_clock.c b/frame/base/bli_clock.c index bd5cd9e82..a4df37e16 100644 --- a/frame/base/bli_clock.c +++ b/frame/base/bli_clock.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_cntl.c b/frame/base/bli_cntl.c index 9020ae8b4..3334cf48e 100644 --- a/frame/base/bli_cntl.c +++ b/frame/base/bli_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_cntl.h b/frame/base/bli_cntl.h index 4e881ea1d..e3968a460 100644 --- a/frame/base/bli_cntl.h +++ b/frame/base/bli_cntl.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_cntx.c b/frame/base/bli_cntx.c index 580bb4905..3d47ff604 100644 --- a/frame/base/bli_cntx.c +++ b/frame/base/bli_cntx.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_cpuid.c b/frame/base/bli_cpuid.c index aacca1da8..d7e954977 100644 --- a/frame/base/bli_cpuid.c +++ b/frame/base/bli_cpuid.c @@ -558,7 +558,7 @@ bool_t bli_cpuid_is_cortexa9 Copyright (C) 2017, The University of Texas at Austin Copyright (C) 2017, Devin Matthews - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_error.c b/frame/base/bli_error.c index 8ed386af5..54677ae27 100644 --- a/frame/base/bli_error.c +++ b/frame/base/bli_error.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_getopt.c b/frame/base/bli_getopt.c index 222223484..184439db5 100644 --- a/frame/base/bli_getopt.c +++ b/frame/base/bli_getopt.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_info.c b/frame/base/bli_info.c index 76844ec23..48baffc77 100644 --- a/frame/base/bli_info.c +++ b/frame/base/bli_info.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_info.h b/frame/base/bli_info.h index 51b589aa5..2addc0c55 100644 --- a/frame/base/bli_info.h +++ b/frame/base/bli_info.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_init.c b/frame/base/bli_init.c index 1180f1c37..db7e5deec 100644 --- a/frame/base/bli_init.c +++ b/frame/base/bli_init.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_malloc.c b/frame/base/bli_malloc.c index 25ebeb1e0..4e9b10ce3 100644 --- a/frame/base/bli_malloc.c +++ b/frame/base/bli_malloc.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_malloc.h b/frame/base/bli_malloc.h index 4e4113bd9..226f732f5 100644 --- a/frame/base/bli_malloc.h +++ b/frame/base/bli_malloc.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_mem.h b/frame/base/bli_mem.h index 5f56f98c0..99977e0c1 100644 --- a/frame/base/bli_mem.h +++ b/frame/base/bli_mem.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_membrk.c b/frame/base/bli_membrk.c index 19b50a52f..762d8c238 100644 --- a/frame/base/bli_membrk.c +++ b/frame/base/bli_membrk.c @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_membrk.h b/frame/base/bli_membrk.h index 4d00eae63..acc026081 100644 --- a/frame/base/bli_membrk.h +++ b/frame/base/bli_membrk.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_memsys.c b/frame/base/bli_memsys.c index 888eb764d..317d3e76d 100644 --- a/frame/base/bli_memsys.c +++ b/frame/base/bli_memsys.c @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_memsys.h b/frame/base/bli_memsys.h index 306819c03..be0d48e35 100644 --- a/frame/base/bli_memsys.h +++ b/frame/base/bli_memsys.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_obj.c b/frame/base/bli_obj.c index 44fdb1f14..2a9b3786e 100644 --- a/frame/base/bli_obj.c +++ b/frame/base/bli_obj.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_pool.c b/frame/base/bli_pool.c index 276169bfe..056c64923 100644 --- a/frame/base/bli_pool.c +++ b/frame/base/bli_pool.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_pool.h b/frame/base/bli_pool.h index 0d39fd7d3..69d990edd 100644 --- a/frame/base/bli_pool.h +++ b/frame/base/bli_pool.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_prune.c b/frame/base/bli_prune.c index 080f66f26..ebe5c2365 100644 --- a/frame/base/bli_prune.c +++ b/frame/base/bli_prune.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_rntm.h b/frame/base/bli_rntm.h index f33c25e36..4e8e74af8 100644 --- a/frame/base/bli_rntm.h +++ b/frame/base/bli_rntm.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_sba.c b/frame/base/bli_sba.c index 1e2d5753f..2e072504c 100644 --- a/frame/base/bli_sba.c +++ b/frame/base/bli_sba.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/base/bli_sba.h b/frame/base/bli_sba.h index cf10834e3..f5e36d759 100644 --- a/frame/base/bli_sba.h +++ b/frame/base/bli_sba.h @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/include/bli_macro_defs.h b/frame/include/bli_macro_defs.h index 1a60aa080..ff23597d1 100644 --- a/frame/include/bli_macro_defs.h +++ b/frame/include/bli_macro_defs.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/include/bli_param_macro_defs.h b/frame/include/bli_param_macro_defs.h index 3c1f74abf..b22949c07 100644 --- a/frame/include/bli_param_macro_defs.h +++ b/frame/include/bli_param_macro_defs.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/include/bli_system.h b/frame/include/bli_system.h index 173bbe1ed..084999ab5 100644 --- a/frame/include/bli_system.h +++ b/frame/include/bli_system.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/include/blis.h b/frame/include/blis.h index 02539bea9..95f9bc5b0 100644 --- a/frame/include/blis.h +++ b/frame/include/blis.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/ind/bli_l3_ind.c b/frame/ind/bli_l3_ind.c index 10897c349..2cb75cae8 100644 --- a/frame/ind/bli_l3_ind.c +++ b/frame/ind/bli_l3_ind.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/ind/oapi/bli_l3_3m4m1m_oapi.c b/frame/ind/oapi/bli_l3_3m4m1m_oapi.c index 087e1beef..3d47e6b51 100644 --- a/frame/ind/oapi/bli_l3_3m4m1m_oapi.c +++ b/frame/ind/oapi/bli_l3_3m4m1m_oapi.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/ind/oapi/bli_l3_ind_oapi.c b/frame/ind/oapi/bli_l3_ind_oapi.c index 213753019..85b4b443e 100644 --- a/frame/ind/oapi/bli_l3_ind_oapi.c +++ b/frame/ind/oapi/bli_l3_ind_oapi.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/ind/oapi/bli_l3_nat_oapi.c b/frame/ind/oapi/bli_l3_nat_oapi.c index 52b7e98ad..470db9c8c 100644 --- a/frame/ind/oapi/bli_l3_nat_oapi.c +++ b/frame/ind/oapi/bli_l3_nat_oapi.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_pthread.c b/frame/thread/bli_pthread.c index 03b44a585..20635c123 100644 --- a/frame/thread/bli_pthread.c +++ b/frame/thread/bli_pthread.c @@ -6,7 +6,7 @@ Copyright (C) 2018, Southern Methodist University Copyright (C) 2018, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrcomm.c b/frame/thread/bli_thrcomm.c index c9698050c..39d2204ee 100644 --- a/frame/thread/bli_thrcomm.c +++ b/frame/thread/bli_thrcomm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrcomm.h b/frame/thread/bli_thrcomm.h index 04bceae2a..b4b7a88db 100644 --- a/frame/thread/bli_thrcomm.h +++ b/frame/thread/bli_thrcomm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrcomm_openmp.c b/frame/thread/bli_thrcomm_openmp.c index 4423f83ff..cffbf14d7 100644 --- a/frame/thread/bli_thrcomm_openmp.c +++ b/frame/thread/bli_thrcomm_openmp.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrcomm_openmp.h b/frame/thread/bli_thrcomm_openmp.h index c7b3b0694..d655bd131 100644 --- a/frame/thread/bli_thrcomm_openmp.h +++ b/frame/thread/bli_thrcomm_openmp.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrcomm_pthreads.c b/frame/thread/bli_thrcomm_pthreads.c index 975c5eb88..e4ea4cfa5 100644 --- a/frame/thread/bli_thrcomm_pthreads.c +++ b/frame/thread/bli_thrcomm_pthreads.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrcomm_single.c b/frame/thread/bli_thrcomm_single.c index 969221e7c..d27a114fe 100644 --- a/frame/thread/bli_thrcomm_single.c +++ b/frame/thread/bli_thrcomm_single.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thread.c b/frame/thread/bli_thread.c index 58ba57e81..bbac4b661 100644 --- a/frame/thread/bli_thread.c +++ b/frame/thread/bli_thread.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thread.h b/frame/thread/bli_thread.h index f186c0f3b..d9bd3d241 100644 --- a/frame/thread/bli_thread.h +++ b/frame/thread/bli_thread.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrinfo.c b/frame/thread/bli_thrinfo.c index 0dcaae2d8..a9d28f615 100644 --- a/frame/thread/bli_thrinfo.c +++ b/frame/thread/bli_thrinfo.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/bli_thrinfo.h b/frame/thread/bli_thrinfo.h index 5903cced0..b06ef2926 100644 --- a/frame/thread/bli_thrinfo.h +++ b/frame/thread/bli_thrinfo.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/old/bli_mutex.h b/frame/thread/old/bli_mutex.h index 95d335622..de9f720e8 100644 --- a/frame/thread/old/bli_mutex.h +++ b/frame/thread/old/bli_mutex.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/old/bli_mutex_openmp.h b/frame/thread/old/bli_mutex_openmp.h index f092d7346..9aaa3c79f 100644 --- a/frame/thread/old/bli_mutex_openmp.h +++ b/frame/thread/old/bli_mutex_openmp.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/old/bli_mutex_pthreads.h b/frame/thread/old/bli_mutex_pthreads.h index 7c87dab47..2053e6128 100644 --- a/frame/thread/old/bli_mutex_pthreads.h +++ b/frame/thread/old/bli_mutex_pthreads.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/thread/old/bli_mutex_single.h b/frame/thread/old/bli_mutex_single.h index 0c8db236b..b57d7bba3 100644 --- a/frame/thread/old/bli_mutex_single.h +++ b/frame/thread/old/bli_mutex_single.h @@ -6,7 +6,7 @@ Copyright (C) 2014, The University of Texas at Austin Copyright (C) 2016, Hewlett Packard Enterprise Development LP - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/frame/util/bli_util_unb_var1.c b/frame/util/bli_util_unb_var1.c index cc0670b58..32197819a 100644 --- a/frame/util/bli_util_unb_var1.c +++ b/frame/util/bli_util_unb_var1.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/haswell/3/bli_gemm_haswell_asm_d6x8.c b/kernels/haswell/3/bli_gemm_haswell_asm_d6x8.c index abb3f0028..6e16287dc 100644 --- a/kernels/haswell/3/bli_gemm_haswell_asm_d6x8.c +++ b/kernels/haswell/3/bli_gemm_haswell_asm_d6x8.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/haswell/3/bli_gemmtrsm_l_haswell_asm_d6x8.c b/kernels/haswell/3/bli_gemmtrsm_l_haswell_asm_d6x8.c index 88514a314..3d69556ff 100644 --- a/kernels/haswell/3/bli_gemmtrsm_l_haswell_asm_d6x8.c +++ b/kernels/haswell/3/bli_gemmtrsm_l_haswell_asm_d6x8.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/haswell/3/bli_gemmtrsm_u_haswell_asm_d6x8.c b/kernels/haswell/3/bli_gemmtrsm_u_haswell_asm_d6x8.c index 89de6c070..ee54d1e3a 100644 --- a/kernels/haswell/3/bli_gemmtrsm_u_haswell_asm_d6x8.c +++ b/kernels/haswell/3/bli_gemmtrsm_u_haswell_asm_d6x8.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/1/bli_amaxv_zen_int.c b/kernels/zen/1/bli_amaxv_zen_int.c index ccf6919cc..496649b50 100644 --- a/kernels/zen/1/bli_amaxv_zen_int.c +++ b/kernels/zen/1/bli_amaxv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc + Copyright (C) 2016 - 2018 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/1/bli_axpyv_zen_int.c b/kernels/zen/1/bli_axpyv_zen_int.c index bd7cec06a..686580b29 100644 --- a/kernels/zen/1/bli_axpyv_zen_int.c +++ b/kernels/zen/1/bli_axpyv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/1/bli_axpyv_zen_int10.c b/kernels/zen/1/bli_axpyv_zen_int10.c index cacbcc6fb..873b7da53 100644 --- a/kernels/zen/1/bli_axpyv_zen_int10.c +++ b/kernels/zen/1/bli_axpyv_zen_int10.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/1/bli_dotv_zen_int.c b/kernels/zen/1/bli_dotv_zen_int.c index 32b43bd10..01022d353 100644 --- a/kernels/zen/1/bli_dotv_zen_int.c +++ b/kernels/zen/1/bli_dotv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/1/bli_dotv_zen_int10.c b/kernels/zen/1/bli_dotv_zen_int10.c index 8874303af..959dc86b5 100644 --- a/kernels/zen/1/bli_dotv_zen_int10.c +++ b/kernels/zen/1/bli_dotv_zen_int10.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/1/bli_dotxv_zen_int.c b/kernels/zen/1/bli_dotxv_zen_int.c index d27225934..99ea51710 100644 --- a/kernels/zen/1/bli_dotxv_zen_int.c +++ b/kernels/zen/1/bli_dotxv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2016 - 2018, Advanced Micro Devices, Inc. + Copyright (C) 2016 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/1/bli_scalv_zen_int.c b/kernels/zen/1/bli_scalv_zen_int.c index 3c58212b0..9f76e88e1 100644 --- a/kernels/zen/1/bli_scalv_zen_int.c +++ b/kernels/zen/1/bli_scalv_zen_int.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/1/bli_scalv_zen_int10.c b/kernels/zen/1/bli_scalv_zen_int10.c index 32812d3df..c02af39dd 100644 --- a/kernels/zen/1/bli_scalv_zen_int10.c +++ b/kernels/zen/1/bli_scalv_zen_int10.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/1f/bli_axpyf_zen_int_8.c b/kernels/zen/1f/bli_axpyf_zen_int_8.c index 13bda01e4..3d424e896 100644 --- a/kernels/zen/1f/bli_axpyf_zen_int_8.c +++ b/kernels/zen/1f/bli_axpyf_zen_int_8.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/1f/bli_dotxf_zen_int_8.c b/kernels/zen/1f/bli_dotxf_zen_int_8.c index 4f0734035..ef75eeda1 100644 --- a/kernels/zen/1f/bli_dotxf_zen_int_8.c +++ b/kernels/zen/1f/bli_dotxf_zen_int_8.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Copyright (C) 2018, The University of Texas at Austin Redistribution and use in source and binary forms, with or without diff --git a/kernels/zen/3/bli_gemm_small.c b/kernels/zen/3/bli_gemm_small.c index dd2c1ce58..1db3b59ea 100644 --- a/kernels/zen/3/bli_gemm_small.c +++ b/kernels/zen/3/bli_gemm_small.c @@ -4,7 +4,7 @@ An object-based framework for developing high-performance BLAS-like libraries. - Copyright (C) 2017 - 2018, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/kernels/zen/3/bli_syrk_small.c b/kernels/zen/3/bli_syrk_small.c index d10114b40..191af2761 100644 --- a/kernels/zen/3/bli_syrk_small.c +++ b/kernels/zen/3/bli_syrk_small.c @@ -4,7 +4,7 @@ BLIS An object-based framework for developing high-performance BLAS-like libraries. -Copyright (C) 2018, Advanced Micro Devices, Inc. +Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/ref_kernels/bli_cntx_ref.c b/ref_kernels/bli_cntx_ref.c index 7b5aa43ef..d97c89bea 100644 --- a/ref_kernels/bli_cntx_ref.c +++ b/ref_kernels/bli_cntx_ref.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/blx_gemm_front.c b/sandbox/ref99/blx_gemm_front.c index bb6ba4a8d..399f750a5 100644 --- a/sandbox/ref99/blx_gemm_front.c +++ b/sandbox/ref99/blx_gemm_front.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/blx_gemm_int.c b/sandbox/ref99/blx_gemm_int.c index c807fc76e..525f72d5d 100644 --- a/sandbox/ref99/blx_gemm_int.c +++ b/sandbox/ref99/blx_gemm_int.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/cntl/blx_gemm_cntl.c b/sandbox/ref99/cntl/blx_gemm_cntl.c index c40410c6f..b26096855 100644 --- a/sandbox/ref99/cntl/blx_gemm_cntl.c +++ b/sandbox/ref99/cntl/blx_gemm_cntl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/oapi/bli_gemmnat.c b/sandbox/ref99/oapi/bli_gemmnat.c index 865c7cff4..e9246338d 100644 --- a/sandbox/ref99/oapi/bli_gemmnat.c +++ b/sandbox/ref99/oapi/bli_gemmnat.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2017, Advanced Micro Devices, Inc. + Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/vars/blx_gemm_blk_var1.c b/sandbox/ref99/vars/blx_gemm_blk_var1.c index ef8c07b1d..dc41b97ff 100644 --- a/sandbox/ref99/vars/blx_gemm_blk_var1.c +++ b/sandbox/ref99/vars/blx_gemm_blk_var1.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/vars/blx_gemm_blk_var2.c b/sandbox/ref99/vars/blx_gemm_blk_var2.c index f272952b0..d7d128c35 100644 --- a/sandbox/ref99/vars/blx_gemm_blk_var2.c +++ b/sandbox/ref99/vars/blx_gemm_blk_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/vars/blx_gemm_ker_var2.c b/sandbox/ref99/vars/blx_gemm_ker_var2.c index 61842411a..10c6b81ad 100644 --- a/sandbox/ref99/vars/blx_gemm_ker_var2.c +++ b/sandbox/ref99/vars/blx_gemm_ker_var2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/vars/blx_gemm_var.h b/sandbox/ref99/vars/blx_gemm_var.h index 32b975d1a..a2a3de9bb 100644 --- a/sandbox/ref99/vars/blx_gemm_var.h +++ b/sandbox/ref99/vars/blx_gemm_var.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/vars/other/blx_gemm_ker_var2rr.c b/sandbox/ref99/vars/other/blx_gemm_ker_var2rr.c index 8a5c1d156..7cbd402e0 100644 --- a/sandbox/ref99/vars/other/blx_gemm_ker_var2rr.c +++ b/sandbox/ref99/vars/other/blx_gemm_ker_var2rr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/sandbox/ref99/vars/other/blx_gemm_ker_var2sl.c b/sandbox/ref99/vars/other/blx_gemm_ker_var2sl.c index 4b0523e37..2d46886b7 100644 --- a/sandbox/ref99/vars/other/blx_gemm_ker_var2sl.c +++ b/sandbox/ref99/vars/other/blx_gemm_ker_var2sl.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/3m4m/Makefile b/test/3m4m/Makefile index f41df2f38..b4ae45bb8 100644 --- a/test/3m4m/Makefile +++ b/test/3m4m/Makefile @@ -5,7 +5,7 @@ # libraries. # # Copyright (C) 2014, The University of Texas at Austin -# Copyright (C) 2018, Advanced Micro Devices, Inc. +# Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are diff --git a/test/3m4m/test_herk.c b/test/3m4m/test_herk.c index d6a1d1fa3..1626bb6fa 100644 --- a/test/3m4m/test_herk.c +++ b/test/3m4m/test_herk.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/3m4m/test_trmm.c b/test/3m4m/test_trmm.c index 6fca5fca3..20bfa11c0 100644 --- a/test/3m4m/test_trmm.c +++ b/test/3m4m/test_trmm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/3m4m/test_trsm.c b/test/3m4m/test_trsm.c index b3f2e33f3..a696a87a6 100644 --- a/test/3m4m/test_trsm.c +++ b/test/3m4m/test_trsm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/Makefile b/test/Makefile index 27d3eaa2c..53d1f9803 100644 --- a/test/Makefile +++ b/test/Makefile @@ -5,7 +5,7 @@ # libraries. # # Copyright (C) 2014, The University of Texas at Austin -# Copyright (C) 2017 - 2018, Advanced Micro Devices, Inc. +# Copyright (C) 2017 - 2019, Advanced Micro Devices, Inc. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are diff --git a/test/test_axpyv.c b/test/test_axpyv.c index c03edb15e..4e584cc87 100644 --- a/test/test_axpyv.c +++ b/test/test_axpyv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/test_dotv.c b/test/test_dotv.c index 98cf4fa2c..f71d2aae9 100644 --- a/test/test_dotv.c +++ b/test/test_dotv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/test/thread_ranges/test_ranges.c b/test/thread_ranges/test_ranges.c index 9bf293ca5..dbbe7f2d8 100644 --- a/test/thread_ranges/test_ranges.c +++ b/test/thread_ranges/test_ranges.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_addm.c b/testsuite/src/test_addm.c index 545f9387b..c957e4dc9 100644 --- a/testsuite/src/test_addm.c +++ b/testsuite/src/test_addm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_addm.h b/testsuite/src/test_addm.h index 815f5db85..0dbdbfa2e 100644 --- a/testsuite/src/test_addm.h +++ b/testsuite/src/test_addm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_addv.c b/testsuite/src/test_addv.c index c394ea1d8..9e216ab4d 100644 --- a/testsuite/src/test_addv.c +++ b/testsuite/src/test_addv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_addv.h b/testsuite/src/test_addv.h index 1b9982e31..eba5a9220 100644 --- a/testsuite/src/test_addv.h +++ b/testsuite/src/test_addv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_amaxv.c b/testsuite/src/test_amaxv.c index d89e08c85..3d58a34ed 100644 --- a/testsuite/src/test_amaxv.c +++ b/testsuite/src/test_amaxv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_amaxv.h b/testsuite/src/test_amaxv.h index 4c382593f..46d87b37f 100644 --- a/testsuite/src/test_amaxv.h +++ b/testsuite/src/test_amaxv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpbyv.c b/testsuite/src/test_axpbyv.c index 24ed4a5ce..2cfdd416c 100644 --- a/testsuite/src/test_axpbyv.c +++ b/testsuite/src/test_axpbyv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpbyv.h b/testsuite/src/test_axpbyv.h index a8fcd2dfa..9b318dba1 100644 --- a/testsuite/src/test_axpbyv.h +++ b/testsuite/src/test_axpbyv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpy2v.c b/testsuite/src/test_axpy2v.c index a834aa6a3..f310d5cb6 100644 --- a/testsuite/src/test_axpy2v.c +++ b/testsuite/src/test_axpy2v.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpy2v.h b/testsuite/src/test_axpy2v.h index dc465792d..c695a643b 100644 --- a/testsuite/src/test_axpy2v.h +++ b/testsuite/src/test_axpy2v.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpyf.c b/testsuite/src/test_axpyf.c index 3bd18ca3e..7572b3a48 100644 --- a/testsuite/src/test_axpyf.c +++ b/testsuite/src/test_axpyf.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpyf.h b/testsuite/src/test_axpyf.h index 179acb9bb..9dd1dadc2 100644 --- a/testsuite/src/test_axpyf.h +++ b/testsuite/src/test_axpyf.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpym.c b/testsuite/src/test_axpym.c index c79866104..9097043a4 100644 --- a/testsuite/src/test_axpym.c +++ b/testsuite/src/test_axpym.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpym.h b/testsuite/src/test_axpym.h index 29819640f..632720284 100644 --- a/testsuite/src/test_axpym.h +++ b/testsuite/src/test_axpym.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpyv.c b/testsuite/src/test_axpyv.c index ff0326fb8..5f3f991ef 100644 --- a/testsuite/src/test_axpyv.c +++ b/testsuite/src/test_axpyv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_axpyv.h b/testsuite/src/test_axpyv.h index a5ce3ea03..c96a9096b 100644 --- a/testsuite/src/test_axpyv.h +++ b/testsuite/src/test_axpyv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_copym.c b/testsuite/src/test_copym.c index 2532a50c7..1aab1d287 100644 --- a/testsuite/src/test_copym.c +++ b/testsuite/src/test_copym.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_copym.h b/testsuite/src/test_copym.h index 2a876ea68..560de0e9a 100644 --- a/testsuite/src/test_copym.h +++ b/testsuite/src/test_copym.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_copyv.c b/testsuite/src/test_copyv.c index 70cb80f23..4350e95ee 100644 --- a/testsuite/src/test_copyv.c +++ b/testsuite/src/test_copyv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_copyv.h b/testsuite/src/test_copyv.h index 1d413f75b..2beb3212d 100644 --- a/testsuite/src/test_copyv.h +++ b/testsuite/src/test_copyv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotaxpyv.c b/testsuite/src/test_dotaxpyv.c index cf5563ec9..28d6c6916 100644 --- a/testsuite/src/test_dotaxpyv.c +++ b/testsuite/src/test_dotaxpyv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotaxpyv.h b/testsuite/src/test_dotaxpyv.h index 742133970..ce82227f4 100644 --- a/testsuite/src/test_dotaxpyv.h +++ b/testsuite/src/test_dotaxpyv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotv.c b/testsuite/src/test_dotv.c index ff9cd2b59..bc4ad54f9 100644 --- a/testsuite/src/test_dotv.c +++ b/testsuite/src/test_dotv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotv.h b/testsuite/src/test_dotv.h index d1b3b0e29..2f000128b 100644 --- a/testsuite/src/test_dotv.h +++ b/testsuite/src/test_dotv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotxaxpyf.c b/testsuite/src/test_dotxaxpyf.c index e85edff17..ca57ca39b 100644 --- a/testsuite/src/test_dotxaxpyf.c +++ b/testsuite/src/test_dotxaxpyf.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotxaxpyf.h b/testsuite/src/test_dotxaxpyf.h index 72b93a637..6bfcd2655 100644 --- a/testsuite/src/test_dotxaxpyf.h +++ b/testsuite/src/test_dotxaxpyf.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotxf.c b/testsuite/src/test_dotxf.c index d73fd0609..eefe2bb77 100644 --- a/testsuite/src/test_dotxf.c +++ b/testsuite/src/test_dotxf.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotxf.h b/testsuite/src/test_dotxf.h index 8940e6a75..06cac584e 100644 --- a/testsuite/src/test_dotxf.h +++ b/testsuite/src/test_dotxf.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotxv.c b/testsuite/src/test_dotxv.c index 76a47a08d..fb677a06d 100644 --- a/testsuite/src/test_dotxv.c +++ b/testsuite/src/test_dotxv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_dotxv.h b/testsuite/src/test_dotxv.h index 02009b5a9..a3e2ca48f 100644 --- a/testsuite/src/test_dotxv.h +++ b/testsuite/src/test_dotxv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemm.c b/testsuite/src/test_gemm.c index 24ecf6d61..2ec117464 100644 --- a/testsuite/src/test_gemm.c +++ b/testsuite/src/test_gemm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemm.h b/testsuite/src/test_gemm.h index f1c41bb95..78364bc24 100644 --- a/testsuite/src/test_gemm.h +++ b/testsuite/src/test_gemm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemm_ukr.c b/testsuite/src/test_gemm_ukr.c index 616532491..8de3f144b 100644 --- a/testsuite/src/test_gemm_ukr.c +++ b/testsuite/src/test_gemm_ukr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemm_ukr.h b/testsuite/src/test_gemm_ukr.h index d20c47c62..cd09ef3f6 100644 --- a/testsuite/src/test_gemm_ukr.h +++ b/testsuite/src/test_gemm_ukr.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemmtrsm_ukr.c b/testsuite/src/test_gemmtrsm_ukr.c index 6d2f028d2..58da5410a 100644 --- a/testsuite/src/test_gemmtrsm_ukr.c +++ b/testsuite/src/test_gemmtrsm_ukr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemmtrsm_ukr.h b/testsuite/src/test_gemmtrsm_ukr.h index 8bf52c4eb..5fd3cc0ba 100644 --- a/testsuite/src/test_gemmtrsm_ukr.h +++ b/testsuite/src/test_gemmtrsm_ukr.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemv.c b/testsuite/src/test_gemv.c index aa10764b0..7df4c3fc8 100644 --- a/testsuite/src/test_gemv.c +++ b/testsuite/src/test_gemv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_gemv.h b/testsuite/src/test_gemv.h index 2de09095e..8e7284486 100644 --- a/testsuite/src/test_gemv.h +++ b/testsuite/src/test_gemv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_ger.c b/testsuite/src/test_ger.c index c611c4661..961247e84 100644 --- a/testsuite/src/test_ger.c +++ b/testsuite/src/test_ger.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_ger.h b/testsuite/src/test_ger.h index f053a73b9..5b75babe6 100644 --- a/testsuite/src/test_ger.h +++ b/testsuite/src/test_ger.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_hemm.c b/testsuite/src/test_hemm.c index 3e2732500..8768af8c7 100644 --- a/testsuite/src/test_hemm.c +++ b/testsuite/src/test_hemm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_hemm.h b/testsuite/src/test_hemm.h index f3ff7b90f..7db76afa1 100644 --- a/testsuite/src/test_hemm.h +++ b/testsuite/src/test_hemm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_hemv.c b/testsuite/src/test_hemv.c index 17204102e..5aca22239 100644 --- a/testsuite/src/test_hemv.c +++ b/testsuite/src/test_hemv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_hemv.h b/testsuite/src/test_hemv.h index 701a79030..e522690d1 100644 --- a/testsuite/src/test_hemv.h +++ b/testsuite/src/test_hemv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_her.c b/testsuite/src/test_her.c index c5ca4b14d..855679cf8 100644 --- a/testsuite/src/test_her.c +++ b/testsuite/src/test_her.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_her.h b/testsuite/src/test_her.h index fed1e45ef..a6aaa55b4 100644 --- a/testsuite/src/test_her.h +++ b/testsuite/src/test_her.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_her2.c b/testsuite/src/test_her2.c index 896497b4e..ee35cc93f 100644 --- a/testsuite/src/test_her2.c +++ b/testsuite/src/test_her2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_her2.h b/testsuite/src/test_her2.h index 6e4d26a64..c2711cfb1 100644 --- a/testsuite/src/test_her2.h +++ b/testsuite/src/test_her2.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_her2k.c b/testsuite/src/test_her2k.c index 067b1c973..c5cf5dbeb 100644 --- a/testsuite/src/test_her2k.c +++ b/testsuite/src/test_her2k.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_her2k.h b/testsuite/src/test_her2k.h index 36382ed9c..a481dac72 100644 --- a/testsuite/src/test_her2k.h +++ b/testsuite/src/test_her2k.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_herk.c b/testsuite/src/test_herk.c index 0db92b4d2..af192f06b 100644 --- a/testsuite/src/test_herk.c +++ b/testsuite/src/test_herk.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_herk.h b/testsuite/src/test_herk.h index 6235cb761..1702bd8b9 100644 --- a/testsuite/src/test_herk.h +++ b/testsuite/src/test_herk.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_libblis.c b/testsuite/src/test_libblis.c index 96c705c9a..780c4b9e6 100644 --- a/testsuite/src/test_libblis.c +++ b/testsuite/src/test_libblis.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_libblis.h b/testsuite/src/test_libblis.h index 4a3dd0ffc..ad1ee3c2c 100644 --- a/testsuite/src/test_libblis.h +++ b/testsuite/src/test_libblis.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_normfm.c b/testsuite/src/test_normfm.c index e8882ed54..8cee0332e 100644 --- a/testsuite/src/test_normfm.c +++ b/testsuite/src/test_normfm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_normfm.h b/testsuite/src/test_normfm.h index b79f6b7bb..a24b5e5ba 100644 --- a/testsuite/src/test_normfm.h +++ b/testsuite/src/test_normfm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_normfv.c b/testsuite/src/test_normfv.c index 1622a2e89..8a473f5fd 100644 --- a/testsuite/src/test_normfv.c +++ b/testsuite/src/test_normfv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_normfv.h b/testsuite/src/test_normfv.h index 2193c43ee..afa535006 100644 --- a/testsuite/src/test_normfv.h +++ b/testsuite/src/test_normfv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_randm.c b/testsuite/src/test_randm.c index f5ef20629..223007dba 100644 --- a/testsuite/src/test_randm.c +++ b/testsuite/src/test_randm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_randm.h b/testsuite/src/test_randm.h index 9c8c87886..e44464962 100644 --- a/testsuite/src/test_randm.h +++ b/testsuite/src/test_randm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_randv.c b/testsuite/src/test_randv.c index 03f98a9b9..951c8c3ec 100644 --- a/testsuite/src/test_randv.c +++ b/testsuite/src/test_randv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_randv.h b/testsuite/src/test_randv.h index 3574c19e5..bb658dfd7 100644 --- a/testsuite/src/test_randv.h +++ b/testsuite/src/test_randv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scal2m.c b/testsuite/src/test_scal2m.c index d6f29f996..3dcff3d78 100644 --- a/testsuite/src/test_scal2m.c +++ b/testsuite/src/test_scal2m.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scal2m.h b/testsuite/src/test_scal2m.h index 3abcd9a14..262723f4e 100644 --- a/testsuite/src/test_scal2m.h +++ b/testsuite/src/test_scal2m.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scal2v.c b/testsuite/src/test_scal2v.c index 7a28479db..94af24502 100644 --- a/testsuite/src/test_scal2v.c +++ b/testsuite/src/test_scal2v.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scal2v.h b/testsuite/src/test_scal2v.h index 3ab6b3c42..75b5cfe4a 100644 --- a/testsuite/src/test_scal2v.h +++ b/testsuite/src/test_scal2v.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scalm.c b/testsuite/src/test_scalm.c index 3e9d5069f..b3f2066e0 100644 --- a/testsuite/src/test_scalm.c +++ b/testsuite/src/test_scalm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scalm.h b/testsuite/src/test_scalm.h index 1723f51dc..3b98617b2 100644 --- a/testsuite/src/test_scalm.h +++ b/testsuite/src/test_scalm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scalv.c b/testsuite/src/test_scalv.c index ef3b980ca..fefb23b4a 100644 --- a/testsuite/src/test_scalv.c +++ b/testsuite/src/test_scalv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_scalv.h b/testsuite/src/test_scalv.h index 9092ae359..144b41675 100644 --- a/testsuite/src/test_scalv.h +++ b/testsuite/src/test_scalv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_setm.c b/testsuite/src/test_setm.c index 630ced831..80cebd64e 100644 --- a/testsuite/src/test_setm.c +++ b/testsuite/src/test_setm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_setm.h b/testsuite/src/test_setm.h index f5cd32aa1..027184031 100644 --- a/testsuite/src/test_setm.h +++ b/testsuite/src/test_setm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_setv.c b/testsuite/src/test_setv.c index a0ed3ee97..10f0348c7 100644 --- a/testsuite/src/test_setv.c +++ b/testsuite/src/test_setv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_setv.h b/testsuite/src/test_setv.h index b2494a17c..4e02d489e 100644 --- a/testsuite/src/test_setv.h +++ b/testsuite/src/test_setv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_subm.c b/testsuite/src/test_subm.c index d28eb2800..b2de8cfad 100644 --- a/testsuite/src/test_subm.c +++ b/testsuite/src/test_subm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_subm.h b/testsuite/src/test_subm.h index c7e7e93ce..e39eff828 100644 --- a/testsuite/src/test_subm.h +++ b/testsuite/src/test_subm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_subv.c b/testsuite/src/test_subv.c index 7d7a107dd..3a48f02a4 100644 --- a/testsuite/src/test_subv.c +++ b/testsuite/src/test_subv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_subv.h b/testsuite/src/test_subv.h index 30fd8bba8..5dbe46589 100644 --- a/testsuite/src/test_subv.h +++ b/testsuite/src/test_subv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_symm.c b/testsuite/src/test_symm.c index f64ad7e76..ded4bb143 100644 --- a/testsuite/src/test_symm.c +++ b/testsuite/src/test_symm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_symm.h b/testsuite/src/test_symm.h index fe960d4fe..bf50bf65d 100644 --- a/testsuite/src/test_symm.h +++ b/testsuite/src/test_symm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_symv.c b/testsuite/src/test_symv.c index c654685df..c99941ea0 100644 --- a/testsuite/src/test_symv.c +++ b/testsuite/src/test_symv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_symv.h b/testsuite/src/test_symv.h index 0a0a833c5..5dba0624c 100644 --- a/testsuite/src/test_symv.h +++ b/testsuite/src/test_symv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syr.c b/testsuite/src/test_syr.c index efdc67b84..b3a6a356d 100644 --- a/testsuite/src/test_syr.c +++ b/testsuite/src/test_syr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syr.h b/testsuite/src/test_syr.h index d616f969b..455e18ff1 100644 --- a/testsuite/src/test_syr.h +++ b/testsuite/src/test_syr.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syr2.c b/testsuite/src/test_syr2.c index e87cd13e5..6b5d72e76 100644 --- a/testsuite/src/test_syr2.c +++ b/testsuite/src/test_syr2.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syr2.h b/testsuite/src/test_syr2.h index 6f0998354..d6c1f3c10 100644 --- a/testsuite/src/test_syr2.h +++ b/testsuite/src/test_syr2.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syr2k.c b/testsuite/src/test_syr2k.c index 9b57502b7..69dfbda7a 100644 --- a/testsuite/src/test_syr2k.c +++ b/testsuite/src/test_syr2k.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syr2k.h b/testsuite/src/test_syr2k.h index 9ceb3befd..edf893c29 100644 --- a/testsuite/src/test_syr2k.h +++ b/testsuite/src/test_syr2k.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syrk.c b/testsuite/src/test_syrk.c index 0ae4a1802..a21bb7af5 100644 --- a/testsuite/src/test_syrk.c +++ b/testsuite/src/test_syrk.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_syrk.h b/testsuite/src/test_syrk.h index e0d461c10..8cad72456 100644 --- a/testsuite/src/test_syrk.h +++ b/testsuite/src/test_syrk.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trmm.c b/testsuite/src/test_trmm.c index 39524b9c1..02505e630 100644 --- a/testsuite/src/test_trmm.c +++ b/testsuite/src/test_trmm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trmm.h b/testsuite/src/test_trmm.h index 0c0b617dd..a84ca1d29 100644 --- a/testsuite/src/test_trmm.h +++ b/testsuite/src/test_trmm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trmm3.c b/testsuite/src/test_trmm3.c index 77e8f2649..bd3937d3b 100644 --- a/testsuite/src/test_trmm3.c +++ b/testsuite/src/test_trmm3.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trmm3.h b/testsuite/src/test_trmm3.h index 6150b7023..ee9490036 100644 --- a/testsuite/src/test_trmm3.h +++ b/testsuite/src/test_trmm3.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trmv.c b/testsuite/src/test_trmv.c index cd1b130cf..85ee1e802 100644 --- a/testsuite/src/test_trmv.c +++ b/testsuite/src/test_trmv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trmv.h b/testsuite/src/test_trmv.h index 185aeb8a3..1fae8331f 100644 --- a/testsuite/src/test_trmv.h +++ b/testsuite/src/test_trmv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trsm.c b/testsuite/src/test_trsm.c index 30435a773..1c90edef5 100644 --- a/testsuite/src/test_trsm.c +++ b/testsuite/src/test_trsm.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trsm.h b/testsuite/src/test_trsm.h index 2738511f2..ee23b2c7a 100644 --- a/testsuite/src/test_trsm.h +++ b/testsuite/src/test_trsm.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trsm_ukr.c b/testsuite/src/test_trsm_ukr.c index 5476e1daf..1f81400c9 100644 --- a/testsuite/src/test_trsm_ukr.c +++ b/testsuite/src/test_trsm_ukr.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trsm_ukr.h b/testsuite/src/test_trsm_ukr.h index 71685bb2a..22c667636 100644 --- a/testsuite/src/test_trsm_ukr.h +++ b/testsuite/src/test_trsm_ukr.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trsv.c b/testsuite/src/test_trsv.c index cb3138c92..15398b193 100644 --- a/testsuite/src/test_trsv.c +++ b/testsuite/src/test_trsv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_trsv.h b/testsuite/src/test_trsv.h index b2e85469d..5f5fa4eb0 100644 --- a/testsuite/src/test_trsv.h +++ b/testsuite/src/test_trsv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_xpbyv.c b/testsuite/src/test_xpbyv.c index fa0abdb82..6b2f21734 100644 --- a/testsuite/src/test_xpbyv.c +++ b/testsuite/src/test_xpbyv.c @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/testsuite/src/test_xpbyv.h b/testsuite/src/test_xpbyv.h index 3b2e7bee2..16eb77216 100644 --- a/testsuite/src/test_xpbyv.h +++ b/testsuite/src/test_xpbyv.h @@ -5,7 +5,7 @@ libraries. Copyright (C) 2014, The University of Texas at Austin - Copyright (C) 2018, Advanced Micro Devices, Inc. + Copyright (C) 2018 - 2019, Advanced Micro Devices, Inc. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are From 3f867c96caea3bbbbeeff1995d90f6cf8c9895fb Mon Sep 17 00:00:00 2001 From: kdevraje Date: Fri, 31 May 2019 12:22:44 +0530 Subject: [PATCH 46/53] When running HPL with pure MPI without DGEMM Threading (Single Threaded BLIS ), making this macro 1 gives best performance.wq Change-Id: I24fd0bf99216f315e49f1c74c44c3feaffd7078d --- config/zen2/bli_cntx_init_zen2.c | 9 ++++++--- config/zen2/bli_family_zen2.h | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/config/zen2/bli_cntx_init_zen2.c b/config/zen2/bli_cntx_init_zen2.c index 20bdcf90b..25eae4866 100644 --- a/config/zen2/bli_cntx_init_zen2.c +++ b/config/zen2/bli_cntx_init_zen2.c @@ -106,12 +106,15 @@ void bli_cntx_init_zen2( cntx_t* cntx ) // s d c z bli_blksz_init_easy( &blkszs[ BLIS_MR ], 6, 6, 3, 3 ); bli_blksz_init_easy( &blkszs[ BLIS_NR ], 16, 8, 8, 4 ); - - +#if AOCL_BLIS_MULTIINSTANCE + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 240, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); +#else bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); - +#endif bli_blksz_init_easy( &blkszs[ BLIS_AF ], 8, 8, -1, -1 ); bli_blksz_init_easy( &blkszs[ BLIS_DF ], 8, 8, -1, -1 ); diff --git a/config/zen2/bli_family_zen2.h b/config/zen2/bli_family_zen2.h index 3422450d8..6f62bd33a 100644 --- a/config/zen2/bli_family_zen2.h +++ b/config/zen2/bli_family_zen2.h @@ -66,5 +66,8 @@ #define D_BLIS_SMALL_MATRIX_THRES_TRSM_DIM_RATIO 22 +//When running HPL with pure MPI without DGEMM Threading (Single Threaded BLIS ), making this macro 1 gives best performance. +#define AOCL_BLIS_MULTIINSTANCE 0 + #endif From b69fb0b74a4756168de270fc9b18f7cf7aa57f17 Mon Sep 17 00:00:00 2001 From: Kiran Varaganti Date: Fri, 31 May 2019 15:14:22 +0530 Subject: [PATCH 47/53] Added back BLIS_ENABLE_ZEN_BLOCK_SIZES macro to zen configuration, this is same as release 1.3. This was added before to improve DGEMM Multithreaded scalability on Naples for when number of threads is greater than 16. By mistake this got deleted in many changes done for 2.0 release, now we are adding this change back., in bli_gemm_front.c - code cleanup Change-Id: I9f5d8225254676a99c6f2b09a0825e545206d0fc --- config/zen/bli_cntx_init_zen.c | 13 +++++++++++-- config/zen/bli_family_zen.h | 2 +- frame/3/gemm/bli_gemm_front.c | 6 ------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/config/zen/bli_cntx_init_zen.c b/config/zen/bli_cntx_init_zen.c index 7f195de24..829cf6d47 100644 --- a/config/zen/bli_cntx_init_zen.c +++ b/config/zen/bli_cntx_init_zen.c @@ -129,17 +129,26 @@ void bli_cntx_init_zen( cntx_t* cntx ) */ // Zen optmized level 3 cache block sizes +#ifdef BLIS_ENABLE_ZEN_BLOCK_SIZES -#if BLIS_ENABLE_SINGLE_INSTANCE_BLOCK_SIZES + #if BLIS_ENABLE_SINGLE_INSTANCE_BLOCK_SIZES bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 510, 144, 72 ); bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 1024, 256, 256 ); bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); -#else + #else + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 240, 144, 72 ); bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 512, 256, 256 ); bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); + + #endif + + bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); + bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); + bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); + #endif diff --git a/config/zen/bli_family_zen.h b/config/zen/bli_family_zen.h index 7721469c2..3f41a53bb 100644 --- a/config/zen/bli_family_zen.h +++ b/config/zen/bli_family_zen.h @@ -42,7 +42,7 @@ #define BLIS_THREAD_MAX_IR 1 #define BLIS_THREAD_MAX_JR 1 - +#define BLIS_ENABLE_ZEN_BLOCK_SIZES #define BLIS_ENABLE_SMALL_MATRIX #define BLIS_ENABLE_SMALL_MATRIX_TRSM diff --git a/frame/3/gemm/bli_gemm_front.c b/frame/3/gemm/bli_gemm_front.c index 026016f26..20928e198 100644 --- a/frame/3/gemm/bli_gemm_front.c +++ b/frame/3/gemm/bli_gemm_front.c @@ -52,13 +52,7 @@ void bli_gemm_front obj_t a_local; obj_t b_local; obj_t c_local; -#if 0 - gint_t M = bli_obj_length( c ); - gint_t N = bli_obj_width( c ); - gint_t K = bli_obj_width( a ); - if( !(M && N && K)) return; -#endif #ifdef BLIS_ENABLE_SMALL_MATRIX // Only handle small problems separately for homogeneous datatypes. From 2acd49b76457635625a01e31c2abc8902b23cf51 Mon Sep 17 00:00:00 2001 From: Meghana Date: Mon, 1 Jul 2019 15:42:38 +0530 Subject: [PATCH 48/53] fix for test failures using AOCC 2.0 Change-Id: If44eaccc64bbe96bbbe1d32279b1b5773aba08d1 --- config/zen/make_defs.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/zen/make_defs.mk b/config/zen/make_defs.mk index a485af27b..d58130b01 100644 --- a/config/zen/make_defs.mk +++ b/config/zen/make_defs.mk @@ -70,7 +70,7 @@ CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 #CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp else ifeq ($(CC_VENDOR),clang) -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-fma4 -mno-tbm -mno-xop -mno-lwp -mllvm -disable-licm-vrp else $(error gcc or clang are required for this configuration.) endif From c7dd6e6cd2f910cbefcdc1e04a5adeb919a23de0 Mon Sep 17 00:00:00 2001 From: Meghana Date: Thu, 4 Jul 2019 09:32:51 +0530 Subject: [PATCH 49/53] Added compiler flags for vanilla clang Change-Id: I13c00b4c0d65bbda4c929848fd48b0ab611952ab --- config/zen/make_defs.mk | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/zen/make_defs.mk b/config/zen/make_defs.mk index d58130b01..e4bd58212 100644 --- a/config/zen/make_defs.mk +++ b/config/zen/make_defs.mk @@ -70,7 +70,10 @@ CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 #CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp else ifeq ($(CC_VENDOR),clang) +#if compiling with AOCC, use these flags CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-fma4 -mno-tbm -mno-xop -mno-lwp -mllvm -disable-licm-vrp +# if compiling with vanilla clang use these flags +#CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-fma4 -mno-tbm -mno-xop -mno-lwp else $(error gcc or clang are required for this configuration.) endif From 1f80858abf5ca220b2998fbe6f9b06c32d3864c3 Mon Sep 17 00:00:00 2001 From: kdevraje Date: Fri, 5 Jul 2019 16:05:11 +0530 Subject: [PATCH 50/53] This checkin solves the dgemm performance issue jira ticket CPUPL 458, as #else was missed during integration, it was always following else path to get the block sizes Change-Id: I0084b5856c2513ab1066c08c15b5086db6532717 --- config/zen/bli_cntx_init_zen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/zen/bli_cntx_init_zen.c b/config/zen/bli_cntx_init_zen.c index 829cf6d47..258d4e92d 100644 --- a/config/zen/bli_cntx_init_zen.c +++ b/config/zen/bli_cntx_init_zen.c @@ -144,7 +144,7 @@ void bli_cntx_init_zen( cntx_t* cntx ) bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 2040, 4080, 4080 ); #endif - +#else bli_blksz_init_easy( &blkszs[ BLIS_MC ], 144, 72, 144, 72 ); bli_blksz_init_easy( &blkszs[ BLIS_KC ], 256, 256, 256, 256 ); bli_blksz_init_easy( &blkszs[ BLIS_NC ], 4080, 4080, 4080, 4080 ); From dcc0ce12fde4c6dca2b4764a1922a2ab19725867 Mon Sep 17 00:00:00 2001 From: Meghana Date: Mon, 22 Jul 2019 17:12:01 +0530 Subject: [PATCH 51/53] Added a global Makefile for AMD architectures in config/zen folder This Makefile(amd_config.mk) has all the flags that are common to EPYC series Change-Id: Ic02c60a8293ccdd37f0f292e631acd198e6895de --- config/zen/amd_config.mk | 82 ++++++++++++++++++++++++++++++++++++++++ config/zen/make_defs.mk | 49 ++++-------------------- config/zen2/make_defs.mk | 42 +++++--------------- 3 files changed, 99 insertions(+), 74 deletions(-) create mode 100644 config/zen/amd_config.mk diff --git a/config/zen/amd_config.mk b/config/zen/amd_config.mk new file mode 100644 index 000000000..56af5485a --- /dev/null +++ b/config/zen/amd_config.mk @@ -0,0 +1,82 @@ +# +# +# BLIS +# An object-based framework for developing high-performance BLAS-like +# libraries. +# +# Copyright (C) 2019, Advanced Micro Devices, Inc. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - Neither the name(s) of the copyright holder(s) nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +# All the common flags for AMD architectures will be added here + +# +# --- Determine the C compiler and related flags --- +# + +# NOTE: The build system will append these variables with various +# general-purpose/configuration-agnostic flags in common.mk. You +# may specify additional flags here as needed. +CPPROCFLAGS := +CMISCFLAGS := +CPICFLAGS := +CWARNFLAGS := + +ifneq ($(DEBUG_TYPE),off) +CDBGFLAGS := -g +endif + +ifeq ($(DEBUG_TYPE),noopt) +COPTFLAGS := -O0 +else +COPTFLAGS := -O3 -fomit-frame-pointer +endif + +# Flags specific to optimized kernels. +CKOPTFLAGS := $(COPTFLAGS) +ifeq ($(CC_VENDOR),gcc) +# gcc 6.0 (clang 4.0) or later: +CKVECFLAGS := -mavx2 -mfpmath=sse -mfma +# gcc 4.9 (clang 3.5) or later: +# possibly add zen-specific instructions: -mclzero -madx -mrdseed -mmwaitx -msha -mxsavec -mxsaves -mclflushopt -mpopcnt +#CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +else +ifeq ($(CC_VENDOR),clang) +#if compiling with AOCC, use these flags +CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +ifeq ($(strip $(shell clang -v |&head -1 |grep -c 'AOCC.LLVM.2.0.0')),1) +CKVECFLAGS += -mllvm -disable-licm-vrp +endif +else +$(error gcc or clang are required for this configuration.) +endif +endif + +# Flags specific to reference kernels. +CROPTFLAGS := $(CKOPTFLAGS) +CRVECFLAGS := $(CKVECFLAGS) + diff --git a/config/zen/make_defs.mk b/config/zen/make_defs.mk index e4bd58212..ea5f0802c 100644 --- a/config/zen/make_defs.mk +++ b/config/zen/make_defs.mk @@ -5,6 +5,7 @@ # libraries. # # Copyright (C) 2014, The University of Texas at Austin +# Copyright (C) 2019, Advanced Micro Devices, Inc. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -32,56 +33,22 @@ # # +# FLAGS specific to zen architecture are added here. +# FLAGS that are common for all the AMD architectures are present in amd_config.mk # Declare the name of the current configuration and add it to the # running list of configurations included by common.mk. THIS_CONFIG := zen #CONFIGS_INCL += $(THIS_CONFIG) -# -# --- Determine the C compiler and related flags --- -# +# Include file containing common flags for all AMD architectures +AMD_CONFIG_FILE := amd_config.mk +AMD_CONFIG_PATH := $(BASE_SHARE_PATH)/config/zen +-include $(AMD_CONFIG_PATH)/$(AMD_CONFIG_FILE) -# NOTE: The build system will append these variables with various -# general-purpose/configuration-agnostic flags in common.mk. You -# may specify additional flags here as needed. -CPPROCFLAGS := -CMISCFLAGS := -CPICFLAGS := -CWARNFLAGS := - -ifneq ($(DEBUG_TYPE),off) -CDBGFLAGS := -g -endif - -ifeq ($(DEBUG_TYPE),noopt) -COPTFLAGS := -O0 -else -COPTFLAGS := -O3 -fomit-frame-pointer -endif - -# Flags specific to optimized kernels. -CKOPTFLAGS := $(COPTFLAGS) ifeq ($(CC_VENDOR),gcc) -# gcc 6.0 (clang 4.0) or later: -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -# gcc 4.9 (clang 3.5) or later: -# possibly add zen-specific instructions: -mclzero -madx -mrdseed -mmwaitx -msha -mxsavec -mxsaves -mclflushopt -mpopcnt -#CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp -else -ifeq ($(CC_VENDOR),clang) -#if compiling with AOCC, use these flags -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-fma4 -mno-tbm -mno-xop -mno-lwp -mllvm -disable-licm-vrp -# if compiling with vanilla clang use these flags -#CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-fma4 -mno-tbm -mno-xop -mno-lwp -else -$(error gcc or clang are required for this configuration.) +CKVECFLAGS += -march=znver1 endif -endif - -# Flags specific to reference kernels. -CROPTFLAGS := $(CKOPTFLAGS) -CRVECFLAGS := $(CKVECFLAGS) # Store all of the variables here to new variables containing the # configuration name. diff --git a/config/zen2/make_defs.mk b/config/zen2/make_defs.mk index 832681ca6..ad4bf6c1e 100644 --- a/config/zen2/make_defs.mk +++ b/config/zen2/make_defs.mk @@ -33,56 +33,32 @@ # # +# FLAGS that are specific to 'zen2' architecture are added here. +# FLAGS that are common for all the AMD architectures are present in config/zen/amd_config.mk +# # Declare the name of the current configuration and add it to the # running list of configurations included by common.mk. THIS_CONFIG := zen2 #CONFIGS_INCL += $(THIS_CONFIG) +# Include file containing common flags for all AMD architectures +AMD_CONFIG_FILE := amd_config.mk +AMD_CONFIG_PATH := $(BASE_SHARE_PATH)/config/zen +-include $(AMD_CONFIG_PATH)/$(AMD_CONFIG_FILE) # # --- Determine the C compiler and related flags --- # - -# NOTE: The build system will append these variables with various -# general-purpose/configuration-agnostic flags in common.mk. You -# may specify additional flags here as needed. -CPPROCFLAGS := -CMISCFLAGS := -CPICFLAGS := -CWARNFLAGS := - -ifneq ($(DEBUG_TYPE),off) -CDBGFLAGS := -g -endif - -ifeq ($(DEBUG_TYPE),noopt) -COPTFLAGS := -O0 -else -COPTFLAGS := -O3 -fomit-frame-pointer -endif - # Flags specific to optimized kernels. -CKOPTFLAGS := $(COPTFLAGS) ifeq ($(CC_VENDOR),gcc) # gcc 9.0 (clang ?) or later: -#CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver2 +#CKVECFLAGS := -march=znver2 # gcc 6.0 (clang 4.0) or later: -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-avx256-split-unaligned-store +CKVECFLAGS += -march=znver1 -mno-avx256-split-unaligned-store # gcc 4.9 (clang 3.5) or later: # possibly add zen-specific instructions: -mclzero -madx -mrdseed -mmwaitx -msha -mxsavec -mxsaves -mclflushopt -mpopcnt #CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp -else -ifeq ($(CC_VENDOR),clang) -CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-fma4 -mno-tbm -mno-xop -mno-lwp -else -$(error gcc or clang are required for this configuration.) endif -endif - -# Flags specific to reference kernels. -CROPTFLAGS := $(CKOPTFLAGS) -CRVECFLAGS := $(CKVECFLAGS) - # Store all of the variables here to new variables containing the # configuration name. $(eval $(call store-make-defs,$(THIS_CONFIG))) From c9486e0c4f82cd9f58f5ceb71c0df039e9970a20 Mon Sep 17 00:00:00 2001 From: Meghana Date: Wed, 24 Jul 2019 09:45:17 +0530 Subject: [PATCH 52/53] code to detect version of gcc and set flags accordingly for zen2 Change-Id: I29b0311d0000dee1a2533ee29941acf53f9e9f34 --- config/zen/amd_config.mk | 2 +- config/zen2/make_defs.mk | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config/zen/amd_config.mk b/config/zen/amd_config.mk index 56af5485a..96340dc73 100644 --- a/config/zen/amd_config.mk +++ b/config/zen/amd_config.mk @@ -66,8 +66,8 @@ CKVECFLAGS := -mavx2 -mfpmath=sse -mfma #CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp else ifeq ($(CC_VENDOR),clang) -#if compiling with AOCC, use these flags CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=znver1 -mno-fma4 -mno-tbm -mno-xop -mno-lwp +#if compiling with AOCC, use these flags ifeq ($(strip $(shell clang -v |&head -1 |grep -c 'AOCC.LLVM.2.0.0')),1) CKVECFLAGS += -mllvm -disable-licm-vrp endif diff --git a/config/zen2/make_defs.mk b/config/zen2/make_defs.mk index ad4bf6c1e..2b191dcb7 100644 --- a/config/zen2/make_defs.mk +++ b/config/zen2/make_defs.mk @@ -52,9 +52,12 @@ AMD_CONFIG_PATH := $(BASE_SHARE_PATH)/config/zen # Flags specific to optimized kernels. ifeq ($(CC_VENDOR),gcc) # gcc 9.0 (clang ?) or later: -#CKVECFLAGS := -march=znver2 +ifeq ($(strip $(shell gcc -dumpversion)),9) +CKVECFLAGS += -march=znver2 # gcc 6.0 (clang 4.0) or later: +else CKVECFLAGS += -march=znver1 -mno-avx256-split-unaligned-store +endif # gcc 4.9 (clang 3.5) or later: # possibly add zen-specific instructions: -mclzero -madx -mrdseed -mmwaitx -msha -mxsavec -mxsaves -mclflushopt -mpopcnt #CKVECFLAGS := -mavx2 -mfpmath=sse -mfma -march=bdver4 -mno-fma4 -mno-tbm -mno-xop -mno-lwp From fdce1a5648d69034fab39943100289323011c36f Mon Sep 17 00:00:00 2001 From: Meghana Date: Wed, 24 Jul 2019 15:04:41 +0530 Subject: [PATCH 53/53] changed gcc version check condition from 'ifeq' to 'if greater or equal' Change-Id: Ie4c461867829bcc113210791bbefb9517e52c226 --- config/zen2/make_defs.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/zen2/make_defs.mk b/config/zen2/make_defs.mk index 2b191dcb7..ac1a0545b 100644 --- a/config/zen2/make_defs.mk +++ b/config/zen2/make_defs.mk @@ -52,7 +52,8 @@ AMD_CONFIG_PATH := $(BASE_SHARE_PATH)/config/zen # Flags specific to optimized kernels. ifeq ($(CC_VENDOR),gcc) # gcc 9.0 (clang ?) or later: -ifeq ($(strip $(shell gcc -dumpversion)),9) +GCC_VERSION := $(strip $(shell gcc -dumpversion)) +ifeq ($(shell test $(GCC_VERSION) -ge 9; echo $$?),0) CKVECFLAGS += -march=znver2 # gcc 6.0 (clang 4.0) or later: else